2011-11-07 12:55:18 +00:00
|
|
|
/*
|
|
|
|
* ***** BEGIN GPL LICENSE BLOCK *****
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version 2
|
|
|
|
* of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software Foundation,
|
|
|
|
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
*
|
|
|
|
* The Original Code is Copyright (C) 2011 Blender Foundation.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Contributor(s): Blender Foundation,
|
|
|
|
* Sergey Sharybin
|
|
|
|
*
|
|
|
|
* ***** END GPL LICENSE BLOCK *****
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* define this to generate PNG images with content of search areas
|
|
|
|
tracking between which failed */
|
|
|
|
#undef DUMP_FAILURE
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
/* define this to generate PNG images with content of search areas
|
|
|
|
on every itteration of tracking */
|
|
|
|
#undef DUMP_ALWAYS
|
|
|
|
|
2011-11-07 12:55:18 +00:00
|
|
|
#include "libmv-capi.h"
|
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
#include "libmv/logging/logging.h"
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
#include "libmv/tracking/track_region.h"
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
#include "libmv/simple_pipeline/callbacks.h"
|
2011-11-07 12:55:18 +00:00
|
|
|
#include "libmv/simple_pipeline/tracks.h"
|
|
|
|
#include "libmv/simple_pipeline/initialize_reconstruction.h"
|
|
|
|
#include "libmv/simple_pipeline/bundle.h"
|
|
|
|
#include "libmv/simple_pipeline/detect.h"
|
|
|
|
#include "libmv/simple_pipeline/pipeline.h"
|
|
|
|
#include "libmv/simple_pipeline/camera_intrinsics.h"
|
2012-04-14 12:02:47 +00:00
|
|
|
#include "libmv/simple_pipeline/modal_solver.h"
|
2013-05-09 16:38:43 +00:00
|
|
|
#include "libmv/simple_pipeline/reconstruction_scale.h"
|
2011-11-07 12:55:18 +00:00
|
|
|
|
|
|
|
#include <stdlib.h>
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
#include <assert.h>
|
2011-11-07 12:55:18 +00:00
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
#if defined(DUMP_FAILURE) || defined (DUMP_ALWAYS)
|
2011-11-07 12:55:18 +00:00
|
|
|
# include <png.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef _MSC_VER
|
|
|
|
# define snprintf _snprintf
|
|
|
|
#endif
|
|
|
|
|
|
|
|
typedef struct libmv_Reconstruction {
|
|
|
|
libmv::EuclideanReconstruction reconstruction;
|
|
|
|
|
|
|
|
/* used for per-track average error calculation after reconstruction */
|
|
|
|
libmv::Tracks tracks;
|
|
|
|
libmv::CameraIntrinsics intrinsics;
|
|
|
|
|
|
|
|
double error;
|
|
|
|
} libmv_Reconstruction;
|
|
|
|
|
|
|
|
typedef struct libmv_Features {
|
|
|
|
int count, margin;
|
|
|
|
libmv::Feature *features;
|
|
|
|
} libmv_Features;
|
|
|
|
|
|
|
|
/* ************ Logging ************ */
|
|
|
|
|
|
|
|
void libmv_initLogging(const char *argv0)
|
|
|
|
{
|
2013-04-15 10:02:33 +00:00
|
|
|
/* Make it so FATAL messages are always print into console */
|
|
|
|
char severity_fatal[32];
|
|
|
|
snprintf(severity_fatal, sizeof(severity_fatal), "%d",
|
|
|
|
google::GLOG_FATAL);
|
|
|
|
|
2011-11-07 12:55:18 +00:00
|
|
|
google::InitGoogleLogging(argv0);
|
|
|
|
google::SetCommandLineOption("logtostderr", "1");
|
|
|
|
google::SetCommandLineOption("v", "0");
|
2013-04-15 10:02:33 +00:00
|
|
|
google::SetCommandLineOption("stderrthreshold", severity_fatal);
|
|
|
|
google::SetCommandLineOption("minloglevel", severity_fatal);
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_startDebugLogging(void)
|
|
|
|
{
|
|
|
|
google::SetCommandLineOption("logtostderr", "1");
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
google::SetCommandLineOption("v", "2");
|
2011-11-07 12:55:18 +00:00
|
|
|
google::SetCommandLineOption("stderrthreshold", "1");
|
|
|
|
google::SetCommandLineOption("minloglevel", "0");
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_setLoggingVerbosity(int verbosity)
|
|
|
|
{
|
|
|
|
char val[10];
|
|
|
|
snprintf(val, sizeof(val), "%d", verbosity);
|
|
|
|
|
|
|
|
google::SetCommandLineOption("v", val);
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:39 +00:00
|
|
|
/* ************ Utility ************ */
|
2012-03-30 10:37:39 +00:00
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
static void floatBufToImage(const float *buf, int width, int height, int channels, libmv::FloatImage *image)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
int x, y, k, a = 0;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
image->Resize(height, width, channels);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
|
|
|
for (y = 0; y < height; y++) {
|
|
|
|
for (x = 0; x < width; x++) {
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
for (k = 0; k < channels; k++) {
|
|
|
|
(*image)(y, x, k) = buf[a++];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void imageToFloatBuf(const libmv::FloatImage *image, int channels, float *buf)
|
|
|
|
{
|
|
|
|
int x, y, k, a = 0;
|
|
|
|
|
|
|
|
for (y = 0; y < image->Height(); y++) {
|
|
|
|
for (x = 0; x < image->Width(); x++) {
|
|
|
|
for (k = 0; k < channels; k++) {
|
|
|
|
buf[a++] = (*image)(y, x, k);
|
|
|
|
}
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
#if defined(DUMP_FAILURE) || defined (DUMP_ALWAYS)
|
2012-12-10 15:18:00 +00:00
|
|
|
static void savePNGImage(png_bytep *row_pointers, int width, int height, int depth, int color_type, char *file_name)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
|
|
|
png_infop info_ptr;
|
|
|
|
png_structp png_ptr;
|
|
|
|
FILE *fp = fopen(file_name, "wb");
|
|
|
|
|
|
|
|
if (!fp)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Initialize stuff */
|
|
|
|
png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
|
|
|
|
info_ptr = png_create_info_struct(png_ptr);
|
|
|
|
|
|
|
|
if (setjmp(png_jmpbuf(png_ptr))) {
|
|
|
|
fclose(fp);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
png_init_io(png_ptr, fp);
|
|
|
|
|
|
|
|
/* write header */
|
|
|
|
if (setjmp(png_jmpbuf(png_ptr))) {
|
|
|
|
fclose(fp);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
png_set_IHDR(png_ptr, info_ptr, width, height,
|
|
|
|
depth, color_type, PNG_INTERLACE_NONE,
|
|
|
|
PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE);
|
|
|
|
|
|
|
|
png_write_info(png_ptr, info_ptr);
|
|
|
|
|
|
|
|
/* write bytes */
|
|
|
|
if (setjmp(png_jmpbuf(png_ptr))) {
|
|
|
|
fclose(fp);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
png_write_image(png_ptr, row_pointers);
|
|
|
|
|
|
|
|
/* end write */
|
|
|
|
if (setjmp(png_jmpbuf(png_ptr))) {
|
|
|
|
fclose(fp);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
png_write_end(png_ptr, NULL);
|
|
|
|
|
|
|
|
fclose(fp);
|
|
|
|
}
|
|
|
|
|
2012-06-12 11:13:53 +00:00
|
|
|
static void saveImage(const char *prefix, libmv::FloatImage image, int x0, int y0)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
|
|
|
int x, y;
|
|
|
|
png_bytep *row_pointers;
|
|
|
|
|
|
|
|
row_pointers= (png_bytep*)malloc(sizeof(png_bytep)*image.Height());
|
|
|
|
|
|
|
|
for (y = 0; y < image.Height(); y++) {
|
|
|
|
row_pointers[y]= (png_bytep)malloc(sizeof(png_byte)*4*image.Width());
|
|
|
|
|
|
|
|
for (x = 0; x < image.Width(); x++) {
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
if (x0 == x && image.Height() - y0 - 1 == y) {
|
2011-11-07 12:55:18 +00:00
|
|
|
row_pointers[y][x*4+0]= 255;
|
|
|
|
row_pointers[y][x*4+1]= 0;
|
|
|
|
row_pointers[y][x*4+2]= 0;
|
|
|
|
row_pointers[y][x*4+3]= 255;
|
|
|
|
}
|
|
|
|
else {
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
float pixel = image(image.Height() - y - 1, x, 0);
|
2011-11-07 12:55:18 +00:00
|
|
|
row_pointers[y][x*4+0]= pixel*255;
|
|
|
|
row_pointers[y][x*4+1]= pixel*255;
|
|
|
|
row_pointers[y][x*4+2]= pixel*255;
|
|
|
|
row_pointers[y][x*4+3]= 255;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
static int a= 0;
|
|
|
|
char buf[128];
|
|
|
|
snprintf(buf, sizeof(buf), "%s_%02d.png", prefix, ++a);
|
|
|
|
savePNGImage(row_pointers, image.Width(), image.Height(), 8, PNG_COLOR_TYPE_RGBA, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (y = 0; y < image.Height(); y++) {
|
|
|
|
free(row_pointers[y]);
|
|
|
|
}
|
|
|
|
free(row_pointers);
|
|
|
|
}
|
|
|
|
|
2012-06-12 11:13:53 +00:00
|
|
|
static void saveBytesImage(const char *prefix, unsigned char *data, int width, int height)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
|
|
|
int x, y;
|
|
|
|
png_bytep *row_pointers;
|
|
|
|
|
|
|
|
row_pointers= (png_bytep*)malloc(sizeof(png_bytep)*height);
|
|
|
|
|
|
|
|
for (y = 0; y < height; y++) {
|
|
|
|
row_pointers[y]= (png_bytep)malloc(sizeof(png_byte)*4*width);
|
|
|
|
|
|
|
|
for (x = 0; x < width; x++) {
|
|
|
|
char pixel = data[width*y+x];
|
|
|
|
row_pointers[y][x*4+0]= pixel;
|
|
|
|
row_pointers[y][x*4+1]= pixel;
|
|
|
|
row_pointers[y][x*4+2]= pixel;
|
|
|
|
row_pointers[y][x*4+3]= 255;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2013-02-25 09:38:59 +00:00
|
|
|
static int a = 0;
|
2011-11-07 12:55:18 +00:00
|
|
|
char buf[128];
|
|
|
|
snprintf(buf, sizeof(buf), "%s_%02d.png", prefix, ++a);
|
|
|
|
savePNGImage(row_pointers, width, height, 8, PNG_COLOR_TYPE_RGBA, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (y = 0; y < height; y++) {
|
|
|
|
free(row_pointers[y]);
|
|
|
|
}
|
|
|
|
free(row_pointers);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
/* ************ Planar tracker ************ */
|
|
|
|
|
|
|
|
/* TrackRegion (new planar tracker) */
|
|
|
|
int libmv_trackRegion(const struct libmv_trackRegionOptions *options,
|
2012-06-11 11:40:54 +00:00
|
|
|
const float *image1, int image1_width, int image1_height,
|
|
|
|
const float *image2, int image2_width, int image2_height,
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
const double *x1, const double *y1,
|
|
|
|
struct libmv_trackRegionResult *result,
|
|
|
|
double *x2, double *y2)
|
|
|
|
{
|
|
|
|
double xx1[5], yy1[5];
|
|
|
|
double xx2[5], yy2[5];
|
|
|
|
bool tracking_result = false;
|
|
|
|
|
|
|
|
/* Convert to doubles for the libmv api. The four corners and the center. */
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
|
|
xx1[i] = x1[i];
|
|
|
|
yy1[i] = y1[i];
|
|
|
|
xx2[i] = x2[i];
|
|
|
|
yy2[i] = y2[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
libmv::TrackRegionOptions track_region_options;
|
2012-06-12 11:13:53 +00:00
|
|
|
libmv::FloatImage image1_mask;
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
switch (options->motion_model) {
|
|
|
|
#define LIBMV_CONVERT(the_model) \
|
|
|
|
case libmv::TrackRegionOptions::the_model: \
|
|
|
|
track_region_options.mode = libmv::TrackRegionOptions::the_model; \
|
|
|
|
break;
|
|
|
|
LIBMV_CONVERT(TRANSLATION)
|
|
|
|
LIBMV_CONVERT(TRANSLATION_ROTATION)
|
|
|
|
LIBMV_CONVERT(TRANSLATION_SCALE)
|
|
|
|
LIBMV_CONVERT(TRANSLATION_ROTATION_SCALE)
|
|
|
|
LIBMV_CONVERT(AFFINE)
|
|
|
|
LIBMV_CONVERT(HOMOGRAPHY)
|
|
|
|
#undef LIBMV_CONVERT
|
|
|
|
}
|
|
|
|
|
|
|
|
track_region_options.minimum_correlation = options->minimum_correlation;
|
|
|
|
track_region_options.max_iterations = options->num_iterations;
|
|
|
|
track_region_options.sigma = options->sigma;
|
|
|
|
track_region_options.num_extra_points = 1;
|
|
|
|
track_region_options.image1_mask = NULL;
|
|
|
|
track_region_options.use_brute_initialization = options->use_brute;
|
|
|
|
track_region_options.use_normalized_intensities = options->use_normalization;
|
|
|
|
|
2012-06-12 11:13:53 +00:00
|
|
|
if (options->image1_mask) {
|
|
|
|
floatBufToImage(options->image1_mask, image1_width, image1_height, 1, &image1_mask);
|
|
|
|
|
|
|
|
track_region_options.image1_mask = &image1_mask;
|
|
|
|
}
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
/* Convert from raw float buffers to libmv's FloatImage. */
|
|
|
|
libmv::FloatImage old_patch, new_patch;
|
2012-06-11 11:40:54 +00:00
|
|
|
floatBufToImage(image1, image1_width, image1_height, 1, &old_patch);
|
|
|
|
floatBufToImage(image2, image2_width, image2_height, 1, &new_patch);
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
|
|
|
|
libmv::TrackRegionResult track_region_result;
|
|
|
|
libmv::TrackRegion(old_patch, new_patch, xx1, yy1, track_region_options, xx2, yy2, &track_region_result);
|
|
|
|
|
|
|
|
/* Convert to floats for the blender api. */
|
|
|
|
for (int i = 0; i < 5; ++i) {
|
|
|
|
x2[i] = xx2[i];
|
|
|
|
y2[i] = yy2[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
/* TODO(keir): Update the termination string with failure details. */
|
|
|
|
if (track_region_result.termination == libmv::TrackRegionResult::PARAMETER_TOLERANCE ||
|
|
|
|
track_region_result.termination == libmv::TrackRegionResult::FUNCTION_TOLERANCE ||
|
|
|
|
track_region_result.termination == libmv::TrackRegionResult::GRADIENT_TOLERANCE ||
|
|
|
|
track_region_result.termination == libmv::TrackRegionResult::NO_CONVERGENCE)
|
|
|
|
{
|
|
|
|
tracking_result = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if defined(DUMP_FAILURE) || defined(DUMP_ALWAYS)
|
|
|
|
#if defined(DUMP_ALWAYS)
|
|
|
|
{
|
|
|
|
#else
|
|
|
|
if (!tracking_result) {
|
|
|
|
#endif
|
|
|
|
saveImage("old_patch", old_patch, x1[4], y1[4]);
|
|
|
|
saveImage("new_patch", new_patch, x2[4], y2[4]);
|
2012-12-10 15:18:00 +00:00
|
|
|
|
|
|
|
if (options->image1_mask)
|
|
|
|
saveImage("mask", image1_mask, x2[4], y2[4]);
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return tracking_result;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_samplePlanarPatch(const float *image, int width, int height,
|
|
|
|
int channels, const double *xs, const double *ys,
|
2012-06-12 11:13:53 +00:00
|
|
|
int num_samples_x, int num_samples_y,
|
|
|
|
const float *mask, float *patch,
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
double *warped_position_x, double *warped_position_y)
|
|
|
|
{
|
2012-06-12 11:13:53 +00:00
|
|
|
libmv::FloatImage libmv_image, libmv_patch, libmv_mask;
|
|
|
|
libmv::FloatImage *libmv_mask_for_sample = NULL;
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
|
|
|
|
floatBufToImage(image, width, height, channels, &libmv_image);
|
|
|
|
|
2012-06-12 11:13:53 +00:00
|
|
|
if (mask) {
|
|
|
|
floatBufToImage(mask, width, height, 1, &libmv_mask);
|
|
|
|
|
|
|
|
libmv_mask_for_sample = &libmv_mask;
|
|
|
|
}
|
|
|
|
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
libmv::SamplePlanarPatch(libmv_image, xs, ys, num_samples_x, num_samples_y,
|
2012-06-12 11:13:53 +00:00
|
|
|
libmv_mask_for_sample, &libmv_patch,
|
|
|
|
warped_position_x, warped_position_y);
|
Planar tracking support for motion tracking
===========================================
Major list of changes done in tomato branch:
- Add a planar tracking implementation to libmv
This adds a new planar tracking implementation to libmv. The
tracker is based on Ceres[1], the new nonlinear minimizer that
myself and Sameer released from Google as open source. Since
the motion model is more involved, the interface is
different than the RegionTracker interface used previously
in Blender.
The start of a C API in libmv-capi.{cpp,h} is also included.
- Migrate from pat_{min,max} for markers to 4 corners representation
Convert markers in the movie clip editor / 2D tracker from using
pat_min and pat_max notation to using the a more general, 4-corner
representation.
There is still considerable porting work to do; in particular
sliding from preview widget does not work correct for rotated
markers.
All other areas should be ported to new representation:
* Added support of sliding individual corners. LMB slide + Ctrl
would scale the whole pattern
* S would scale the whole marker, S-S would scale pattern only
* Added support of marker's rotation which is currently rotates
only patterns around their centers or all markers around median,
Rotation or other non-translation/scaling transformation of search
area doesn't make sense.
* Track Preview widget would display transformed pattern which
libmv actually operates with.
- "Efficient Second-order Minimization" for the planar tracker
This implements the "Efficient Second-order Minimization"
scheme, as supported by the existing translation tracker.
This increases the amount of per-iteration work, but
decreases the number of iterations required to converge and
also increases the size of the basin of attraction for the
optimization.
- Remove the use of the legacy RegionTracker API from Blender,
and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:
* Do a brute-force initialization of tracking similar to "Hybrid"
mode in the stable release, but using all floats. This is slower
but more accurate. It is still necessary to evaluate if the
performance loss is worth it. In particular, this change is
necessary to support high bit depth imagery.
* Add support for masks over the search window. This is a step
towards supporting user-defined tracker masks. The tracker masks
will make it easy for users to make a mask for e.g. a ball.
Not exposed into interface yet/
* Add Pearson product moment correlation coefficient checking (aka
"Correlation" in the UI. This causes tracking failure if the
tracked patch is not linearly related to the template.
* Add support for warping a few points in addition to the supplied
points. This is useful because the tracking code deliberately
does not expose the underlying warp representation. Instead,
warps are specified in an aparametric way via the correspondences.
- Replace the old style tracker configuration panel with the
new planar tracking panel. From a users perspective, this means:
* The old "tracking algorithm" picker is gone. There is only 1
algorithm now. We may revisit this later, but I would much
prefer to have only 1 algorithm. So far no optimization work
has been done so the speed is not there yet.
* There is now a dropdown to select the motion model. Choices:
* Translation
* Translation, rotation
* Translation, scale
* Translation, rotation, scale
* Affine
* Perspective
* The old "Hybrid" mode is gone; instead there is a toggle to
enable or disable translation-only tracker initialization. This
is the equivalent of the hyrbid mode before, but rewritten to work
with the new planar tracking modes.
* The pyramid levels setting is gone. At a future date, the planar
tracker will decide to use pyramids or not automatically. The
pyramid setting was ultimately a mistake; with the brute force
initialization it is unnecessary.
- Add light-normalized tracking
Added the ability to normalize patterns by their average value while
tracking, to make them invariant to global illumination changes.
Additional details could be found at wiki page [2]
[1] http://code.google.com/p/ceres-solver
[2] http://wiki.blender.org/index.php/Dev:Ref/Release_Notes/2.64/Motion_Tracker
2012-06-10 15:28:19 +00:00
|
|
|
|
|
|
|
imageToFloatBuf(&libmv_patch, channels, patch);
|
|
|
|
}
|
|
|
|
|
2011-11-07 12:55:18 +00:00
|
|
|
/* ************ Tracks ************ */
|
|
|
|
|
|
|
|
libmv_Tracks *libmv_tracksNew(void)
|
|
|
|
{
|
|
|
|
libmv::Tracks *libmv_tracks = new libmv::Tracks();
|
|
|
|
|
|
|
|
return (libmv_Tracks *)libmv_tracks;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y)
|
|
|
|
{
|
|
|
|
((libmv::Tracks*)libmv_tracks)->Insert(image, track, x, y);
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_tracksDestroy(libmv_Tracks *libmv_tracks)
|
|
|
|
{
|
|
|
|
delete (libmv::Tracks*)libmv_tracks;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ************ Reconstruction solver ************ */
|
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
class ReconstructUpdateCallback : public libmv::ProgressUpdateCallback {
|
|
|
|
public:
|
|
|
|
ReconstructUpdateCallback(reconstruct_progress_update_cb progress_update_callback,
|
|
|
|
void *callback_customdata)
|
|
|
|
{
|
|
|
|
progress_update_callback_ = progress_update_callback;
|
|
|
|
callback_customdata_ = callback_customdata;
|
|
|
|
}
|
|
|
|
|
|
|
|
void invoke(double progress, const char *message)
|
|
|
|
{
|
|
|
|
if(progress_update_callback_) {
|
|
|
|
progress_update_callback_(callback_customdata_, progress, message);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
protected:
|
|
|
|
reconstruct_progress_update_cb progress_update_callback_;
|
|
|
|
void *callback_customdata_;
|
|
|
|
};
|
|
|
|
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
int libmv_refineParametersAreValid(int parameters) {
|
|
|
|
return (parameters == (LIBMV_REFINE_FOCAL_LENGTH)) ||
|
|
|
|
(parameters == (LIBMV_REFINE_FOCAL_LENGTH |
|
|
|
|
LIBMV_REFINE_PRINCIPAL_POINT)) ||
|
|
|
|
(parameters == (LIBMV_REFINE_FOCAL_LENGTH |
|
|
|
|
LIBMV_REFINE_PRINCIPAL_POINT |
|
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K1 |
|
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K2)) ||
|
|
|
|
(parameters == (LIBMV_REFINE_FOCAL_LENGTH |
|
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K1 |
|
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K2)) ||
|
|
|
|
(parameters == (LIBMV_REFINE_FOCAL_LENGTH |
|
2013-02-26 11:46:38 +00:00
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K1)) ||
|
|
|
|
(parameters == (LIBMV_REFINE_RADIAL_DISTORTION_K1 |
|
|
|
|
LIBMV_REFINE_RADIAL_DISTORTION_K2));
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
}
|
|
|
|
|
2012-11-05 19:42:27 +00:00
|
|
|
static void libmv_solveRefineIntrinsics(libmv::Tracks *tracks, libmv::CameraIntrinsics *intrinsics,
|
2012-04-14 12:02:47 +00:00
|
|
|
libmv::EuclideanReconstruction *reconstruction, int refine_intrinsics,
|
2013-02-28 14:24:42 +00:00
|
|
|
reconstruct_progress_update_cb progress_update_callback, void *callback_customdata,
|
|
|
|
int bundle_constraints = libmv::BUNDLE_NO_CONSTRAINTS)
|
2012-04-14 12:02:47 +00:00
|
|
|
{
|
|
|
|
/* only a few combinations are supported but trust the caller */
|
|
|
|
int libmv_refine_flags = 0;
|
|
|
|
|
|
|
|
if (refine_intrinsics & LIBMV_REFINE_FOCAL_LENGTH) {
|
|
|
|
libmv_refine_flags |= libmv::BUNDLE_FOCAL_LENGTH;
|
|
|
|
}
|
|
|
|
if (refine_intrinsics & LIBMV_REFINE_PRINCIPAL_POINT) {
|
|
|
|
libmv_refine_flags |= libmv::BUNDLE_PRINCIPAL_POINT;
|
|
|
|
}
|
|
|
|
if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K1) {
|
|
|
|
libmv_refine_flags |= libmv::BUNDLE_RADIAL_K1;
|
|
|
|
}
|
|
|
|
if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K2) {
|
|
|
|
libmv_refine_flags |= libmv::BUNDLE_RADIAL_K2;
|
|
|
|
}
|
|
|
|
|
|
|
|
progress_update_callback(callback_customdata, 1.0, "Refining solution");
|
|
|
|
|
|
|
|
libmv::EuclideanBundleCommonIntrinsics(*(libmv::Tracks *)tracks, libmv_refine_flags,
|
2013-02-28 14:24:42 +00:00
|
|
|
reconstruction, intrinsics, bundle_constraints);
|
2012-04-14 12:02:47 +00:00
|
|
|
}
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
static void cameraIntrinsicsFromOptions(libmv::CameraIntrinsics *camera_intrinsics,
|
|
|
|
libmv_cameraIntrinsicsOptions *camera_intrinsics_options)
|
|
|
|
{
|
|
|
|
camera_intrinsics->SetFocalLength(camera_intrinsics_options->focal_length,
|
|
|
|
camera_intrinsics_options->focal_length);
|
|
|
|
|
|
|
|
camera_intrinsics->SetPrincipalPoint(camera_intrinsics_options->principal_point_x,
|
|
|
|
camera_intrinsics_options->principal_point_y);
|
|
|
|
|
|
|
|
camera_intrinsics->SetRadialDistortion(camera_intrinsics_options->k1,
|
|
|
|
camera_intrinsics_options->k2,
|
|
|
|
camera_intrinsics_options->k3);
|
2012-12-10 16:38:28 +00:00
|
|
|
|
|
|
|
camera_intrinsics->SetImageSize(camera_intrinsics_options->image_width,
|
|
|
|
camera_intrinsics_options->image_height);
|
2012-12-10 16:38:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static libmv::Tracks getNormalizedTracks(libmv::Tracks *tracks, libmv::CameraIntrinsics *camera_intrinsics)
|
|
|
|
{
|
|
|
|
libmv::vector<libmv::Marker> markers = tracks->AllMarkers();
|
|
|
|
|
|
|
|
for (int i = 0; i < markers.size(); ++i) {
|
|
|
|
camera_intrinsics->InvertIntrinsics(markers[i].x, markers[i].y,
|
|
|
|
&(markers[i].x), &(markers[i].y));
|
|
|
|
}
|
|
|
|
|
|
|
|
return libmv::Tracks(markers);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void finishReconstruction(libmv::Tracks *tracks, libmv::CameraIntrinsics *camera_intrinsics,
|
|
|
|
libmv_Reconstruction *libmv_reconstruction,
|
|
|
|
reconstruct_progress_update_cb progress_update_callback,
|
|
|
|
void *callback_customdata)
|
|
|
|
{
|
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
|
|
|
|
|
|
|
/* reprojection error calculation */
|
|
|
|
progress_update_callback(callback_customdata, 1.0, "Finishing solution");
|
|
|
|
libmv_reconstruction->tracks = *tracks;
|
|
|
|
libmv_reconstruction->error = libmv::EuclideanReprojectionError(*tracks, *reconstruction, *camera_intrinsics);
|
|
|
|
}
|
|
|
|
|
|
|
|
libmv_Reconstruction *libmv_solveReconstruction(libmv_Tracks *libmv_tracks,
|
|
|
|
libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options,
|
|
|
|
libmv_reconstructionOptions *libmv_reconstruction_options,
|
|
|
|
reconstruct_progress_update_cb progress_update_callback,
|
|
|
|
void *callback_customdata)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
|
|
|
libmv_Reconstruction *libmv_reconstruction = new libmv_Reconstruction();
|
2012-12-10 16:38:13 +00:00
|
|
|
|
|
|
|
libmv::Tracks *tracks = ((libmv::Tracks *) libmv_tracks);
|
2011-11-07 12:55:18 +00:00
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
2012-12-10 16:38:13 +00:00
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = &libmv_reconstruction->intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
ReconstructUpdateCallback update_callback =
|
|
|
|
ReconstructUpdateCallback(progress_update_callback, callback_customdata);
|
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
cameraIntrinsicsFromOptions(camera_intrinsics, libmv_camera_intrinsics_options);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
/* Invert the camera intrinsics */
|
|
|
|
libmv::Tracks normalized_tracks = getNormalizedTracks(tracks, camera_intrinsics);
|
2012-11-05 19:42:27 +00:00
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
/* actual reconstruction */
|
|
|
|
libmv::ReconstructionOptions reconstruction_options;
|
|
|
|
reconstruction_options.success_threshold = libmv_reconstruction_options->success_threshold;
|
|
|
|
reconstruction_options.use_fallback_reconstruction = libmv_reconstruction_options->use_fallback_reconstruction;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
int keyframe1 = libmv_reconstruction_options->keyframe1,
|
|
|
|
keyframe2 = libmv_reconstruction_options->keyframe2;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
LG << "frames to init from: " << keyframe1 << " " << keyframe2;
|
2012-12-10 16:38:13 +00:00
|
|
|
|
2011-11-07 12:55:18 +00:00
|
|
|
libmv::vector<libmv::Marker> keyframe_markers =
|
|
|
|
normalized_tracks.MarkersForTracksInBothImages(keyframe1, keyframe2);
|
2012-12-10 16:38:13 +00:00
|
|
|
|
2011-11-28 13:49:42 +00:00
|
|
|
LG << "number of markers for init: " << keyframe_markers.size();
|
|
|
|
|
|
|
|
update_callback.invoke(0, "Initial reconstruction");
|
2011-11-07 12:55:18 +00:00
|
|
|
|
|
|
|
libmv::EuclideanReconstructTwoFrames(keyframe_markers, reconstruction);
|
|
|
|
libmv::EuclideanBundle(normalized_tracks, reconstruction);
|
2012-11-05 19:42:27 +00:00
|
|
|
libmv::EuclideanCompleteReconstruction(reconstruction_options, normalized_tracks,
|
|
|
|
reconstruction, &update_callback);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
/* refinement */
|
|
|
|
if (libmv_reconstruction_options->refine_intrinsics) {
|
|
|
|
libmv_solveRefineIntrinsics((libmv::Tracks *)tracks, camera_intrinsics, reconstruction,
|
|
|
|
libmv_reconstruction_options->refine_intrinsics,
|
|
|
|
progress_update_callback, callback_customdata);
|
2012-04-14 12:02:47 +00:00
|
|
|
}
|
2011-11-28 13:49:42 +00:00
|
|
|
|
2013-05-09 16:38:43 +00:00
|
|
|
/* set reconstruction scale to unity */
|
|
|
|
libmv::EuclideanScaleToUnity(reconstruction);
|
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
/* finish reconstruction */
|
|
|
|
finishReconstruction(tracks, camera_intrinsics, libmv_reconstruction,
|
|
|
|
progress_update_callback, callback_customdata);
|
2012-04-14 12:02:47 +00:00
|
|
|
|
|
|
|
return (libmv_Reconstruction *)libmv_reconstruction;
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
struct libmv_Reconstruction *libmv_solveModal(struct libmv_Tracks *libmv_tracks,
|
|
|
|
libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options,
|
2013-02-28 14:24:42 +00:00
|
|
|
libmv_reconstructionOptions *libmv_reconstruction_options,
|
2012-12-10 16:38:13 +00:00
|
|
|
reconstruct_progress_update_cb progress_update_callback,
|
|
|
|
void *callback_customdata)
|
2012-04-14 12:02:47 +00:00
|
|
|
{
|
|
|
|
libmv_Reconstruction *libmv_reconstruction = new libmv_Reconstruction();
|
2012-12-10 16:38:13 +00:00
|
|
|
|
|
|
|
libmv::Tracks *tracks = ((libmv::Tracks *) libmv_tracks);
|
2012-04-14 12:02:47 +00:00
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
2012-12-10 16:38:13 +00:00
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = &libmv_reconstruction->intrinsics;
|
2012-04-14 12:02:47 +00:00
|
|
|
|
|
|
|
ReconstructUpdateCallback update_callback =
|
|
|
|
ReconstructUpdateCallback(progress_update_callback, callback_customdata);
|
|
|
|
|
2012-12-10 16:38:13 +00:00
|
|
|
cameraIntrinsicsFromOptions(camera_intrinsics, libmv_camera_intrinsics_options);
|
2012-04-14 12:02:47 +00:00
|
|
|
|
2013-02-28 14:24:42 +00:00
|
|
|
/* Invert the camera intrinsics. */
|
2012-12-10 16:38:13 +00:00
|
|
|
libmv::Tracks normalized_tracks = getNormalizedTracks(tracks, camera_intrinsics);
|
2012-04-14 12:02:47 +00:00
|
|
|
|
2013-02-28 14:24:42 +00:00
|
|
|
/* Actual reconstruction. */
|
2012-04-14 12:02:47 +00:00
|
|
|
libmv::ModalSolver(normalized_tracks, reconstruction, &update_callback);
|
2011-11-28 13:49:42 +00:00
|
|
|
|
2013-02-28 14:24:42 +00:00
|
|
|
libmv::CameraIntrinsics empty_intrinsics;
|
|
|
|
libmv::EuclideanBundleCommonIntrinsics(normalized_tracks,
|
|
|
|
libmv::BUNDLE_NO_INTRINSICS,
|
|
|
|
reconstruction,
|
|
|
|
&empty_intrinsics,
|
|
|
|
libmv::BUNDLE_NO_TRANSLATION);
|
|
|
|
|
|
|
|
/* Refinement. */
|
|
|
|
if (libmv_reconstruction_options->refine_intrinsics) {
|
|
|
|
libmv_solveRefineIntrinsics((libmv::Tracks *)tracks, camera_intrinsics, reconstruction,
|
|
|
|
libmv_reconstruction_options->refine_intrinsics,
|
|
|
|
progress_update_callback, callback_customdata,
|
|
|
|
libmv::BUNDLE_NO_TRANSLATION);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Finish reconstruction. */
|
2012-12-10 16:38:13 +00:00
|
|
|
finishReconstruction(tracks, camera_intrinsics, libmv_reconstruction,
|
|
|
|
progress_update_callback, callback_customdata);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
|
|
|
return (libmv_Reconstruction *)libmv_reconstruction;
|
|
|
|
}
|
|
|
|
|
|
|
|
int libmv_reporojectionPointForTrack(libmv_Reconstruction *libmv_reconstruction, int track, double pos[3])
|
|
|
|
{
|
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
|
|
|
libmv::EuclideanPoint *point = reconstruction->PointForTrack(track);
|
|
|
|
|
|
|
|
if(point) {
|
|
|
|
pos[0] = point->X[0];
|
|
|
|
pos[1] = point->X[2];
|
|
|
|
pos[2] = point->X[1];
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static libmv::Marker ProjectMarker(const libmv::EuclideanPoint &point, const libmv::EuclideanCamera &camera,
|
|
|
|
const libmv::CameraIntrinsics &intrinsics) {
|
|
|
|
libmv::Vec3 projected = camera.R * point.X + camera.t;
|
|
|
|
projected /= projected(2);
|
|
|
|
|
|
|
|
libmv::Marker reprojected_marker;
|
|
|
|
intrinsics.ApplyIntrinsics(projected(0), projected(1), &reprojected_marker.x, &reprojected_marker.y);
|
|
|
|
|
|
|
|
reprojected_marker.image = camera.image;
|
|
|
|
reprojected_marker.track = point.track;
|
|
|
|
|
|
|
|
return reprojected_marker;
|
|
|
|
}
|
|
|
|
|
|
|
|
double libmv_reporojectionErrorForTrack(libmv_Reconstruction *libmv_reconstruction, int track)
|
|
|
|
{
|
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
|
|
|
libmv::CameraIntrinsics *intrinsics = &libmv_reconstruction->intrinsics;
|
2011-12-05 07:56:31 +00:00
|
|
|
libmv::vector<libmv::Marker> markers = libmv_reconstruction->tracks.MarkersForTrack(track);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
|
|
|
int num_reprojected = 0;
|
|
|
|
double total_error = 0.0;
|
|
|
|
|
|
|
|
for (int i = 0; i < markers.size(); ++i) {
|
|
|
|
const libmv::EuclideanCamera *camera = reconstruction->CameraForImage(markers[i].image);
|
|
|
|
const libmv::EuclideanPoint *point = reconstruction->PointForTrack(markers[i].track);
|
|
|
|
|
|
|
|
if (!camera || !point) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
num_reprojected++;
|
|
|
|
|
|
|
|
libmv::Marker reprojected_marker = ProjectMarker(*point, *camera, *intrinsics);
|
|
|
|
double ex = reprojected_marker.x - markers[i].x;
|
|
|
|
double ey = reprojected_marker.y - markers[i].y;
|
|
|
|
|
|
|
|
total_error += sqrt(ex*ex + ey*ey);
|
|
|
|
}
|
|
|
|
|
|
|
|
return total_error / num_reprojected;
|
|
|
|
}
|
|
|
|
|
|
|
|
double libmv_reporojectionErrorForImage(libmv_Reconstruction *libmv_reconstruction, int image)
|
|
|
|
{
|
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
|
|
|
libmv::CameraIntrinsics *intrinsics = &libmv_reconstruction->intrinsics;
|
|
|
|
libmv::vector<libmv::Marker> markers = libmv_reconstruction->tracks.MarkersInImage(image);
|
|
|
|
const libmv::EuclideanCamera *camera = reconstruction->CameraForImage(image);
|
|
|
|
int num_reprojected = 0;
|
|
|
|
double total_error = 0.0;
|
|
|
|
|
|
|
|
if (!camera)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for (int i = 0; i < markers.size(); ++i) {
|
|
|
|
const libmv::EuclideanPoint *point = reconstruction->PointForTrack(markers[i].track);
|
|
|
|
|
|
|
|
if (!point) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
num_reprojected++;
|
|
|
|
|
|
|
|
libmv::Marker reprojected_marker = ProjectMarker(*point, *camera, *intrinsics);
|
|
|
|
double ex = reprojected_marker.x - markers[i].x;
|
|
|
|
double ey = reprojected_marker.y - markers[i].y;
|
|
|
|
|
|
|
|
total_error += sqrt(ex*ex + ey*ey);
|
|
|
|
}
|
|
|
|
|
|
|
|
return total_error / num_reprojected;
|
|
|
|
}
|
|
|
|
|
|
|
|
int libmv_reporojectionCameraForImage(libmv_Reconstruction *libmv_reconstruction, int image, double mat[4][4])
|
|
|
|
{
|
|
|
|
libmv::EuclideanReconstruction *reconstruction = &libmv_reconstruction->reconstruction;
|
|
|
|
libmv::EuclideanCamera *camera = reconstruction->CameraForImage(image);
|
|
|
|
|
|
|
|
if(camera) {
|
|
|
|
for (int j = 0; j < 3; ++j) {
|
|
|
|
for (int k = 0; k < 3; ++k) {
|
|
|
|
int l = k;
|
|
|
|
|
|
|
|
if (k == 1) l = 2;
|
|
|
|
else if (k == 2) l = 1;
|
|
|
|
|
|
|
|
if (j == 2) mat[j][l] = -camera->R(j,k);
|
|
|
|
else mat[j][l] = camera->R(j,k);
|
|
|
|
}
|
|
|
|
mat[j][3]= 0.0;
|
|
|
|
}
|
|
|
|
|
|
|
|
libmv::Vec3 optical_center = -camera->R.transpose() * camera->t;
|
|
|
|
|
|
|
|
mat[3][0] = optical_center(0);
|
|
|
|
mat[3][1] = optical_center(2);
|
|
|
|
mat[3][2] = optical_center(1);
|
|
|
|
|
|
|
|
mat[3][3]= 1.0;
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
double libmv_reprojectionError(libmv_Reconstruction *libmv_reconstruction)
|
|
|
|
{
|
|
|
|
return libmv_reconstruction->error;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_destroyReconstruction(libmv_Reconstruction *libmv_reconstruction)
|
|
|
|
{
|
|
|
|
delete libmv_reconstruction;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ************ feature detector ************ */
|
|
|
|
|
|
|
|
struct libmv_Features *libmv_detectFeaturesFAST(unsigned char *data, int width, int height, int stride,
|
|
|
|
int margin, int min_trackness, int min_distance)
|
|
|
|
{
|
|
|
|
libmv::Feature *features = NULL;
|
|
|
|
std::vector<libmv::Feature> v;
|
|
|
|
libmv_Features *libmv_features = new libmv_Features();
|
|
|
|
int i= 0, count;
|
|
|
|
|
|
|
|
if(margin) {
|
|
|
|
data += margin*stride+margin;
|
|
|
|
width -= 2*margin;
|
|
|
|
height -= 2*margin;
|
|
|
|
}
|
|
|
|
|
|
|
|
v = libmv::DetectFAST(data, width, height, stride, min_trackness, min_distance);
|
|
|
|
|
|
|
|
count = v.size();
|
|
|
|
|
|
|
|
if(count) {
|
|
|
|
features= new libmv::Feature[count];
|
|
|
|
|
|
|
|
for(std::vector<libmv::Feature>::iterator it = v.begin(); it != v.end(); it++) {
|
|
|
|
features[i++]= *it;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
libmv_features->features = features;
|
|
|
|
libmv_features->count = count;
|
|
|
|
libmv_features->margin = margin;
|
|
|
|
|
|
|
|
return (libmv_Features *)libmv_features;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct libmv_Features *libmv_detectFeaturesMORAVEC(unsigned char *data, int width, int height, int stride,
|
|
|
|
int margin, int count, int min_distance)
|
|
|
|
{
|
|
|
|
libmv::Feature *features = NULL;
|
|
|
|
libmv_Features *libmv_features = new libmv_Features;
|
|
|
|
|
|
|
|
if(count) {
|
|
|
|
if(margin) {
|
|
|
|
data += margin*stride+margin;
|
|
|
|
width -= 2*margin;
|
|
|
|
height -= 2*margin;
|
|
|
|
}
|
|
|
|
|
|
|
|
features = new libmv::Feature[count];
|
|
|
|
libmv::DetectMORAVEC(data, stride, width, height, features, &count, min_distance, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
libmv_features->count = count;
|
|
|
|
libmv_features->margin = margin;
|
|
|
|
libmv_features->features = features;
|
|
|
|
|
|
|
|
return libmv_features;
|
|
|
|
}
|
|
|
|
|
|
|
|
int libmv_countFeatures(struct libmv_Features *libmv_features)
|
|
|
|
{
|
|
|
|
return libmv_features->count;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_getFeature(struct libmv_Features *libmv_features, int number, double *x, double *y, double *score, double *size)
|
|
|
|
{
|
|
|
|
libmv::Feature feature= libmv_features->features[number];
|
|
|
|
|
|
|
|
*x = feature.x + libmv_features->margin;
|
|
|
|
*y = feature.y + libmv_features->margin;
|
|
|
|
*score = feature.score;
|
|
|
|
*size = feature.size;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_destroyFeatures(struct libmv_Features *libmv_features)
|
|
|
|
{
|
|
|
|
if(libmv_features->features)
|
|
|
|
delete [] libmv_features->features;
|
|
|
|
|
|
|
|
delete libmv_features;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ************ camera intrinsics ************ */
|
|
|
|
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
struct libmv_CameraIntrinsics *libmv_ReconstructionExtractIntrinsics(struct libmv_Reconstruction *libmv_Reconstruction) {
|
2011-12-05 07:56:31 +00:00
|
|
|
return (struct libmv_CameraIntrinsics *)&libmv_Reconstruction->intrinsics;
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
}
|
|
|
|
|
2013-03-15 11:59:46 +00:00
|
|
|
struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNewEmpty(void)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
|
|
|
|
|
|
|
|
return (struct libmv_CameraIntrinsics *) camera_intrinsics;
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNew(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
2012-12-10 16:38:28 +00:00
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
cameraIntrinsicsFromOptions(camera_intrinsics, libmv_camera_intrinsics_options);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
return (struct libmv_CameraIntrinsics *) camera_intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsCopy(struct libmv_CameraIntrinsics *libmvIntrinsics)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *orig_intrinsics = (libmv::CameraIntrinsics *) libmvIntrinsics;
|
|
|
|
libmv::CameraIntrinsics *new_intrinsics= new libmv::CameraIntrinsics(*orig_intrinsics);
|
|
|
|
|
|
|
|
return (struct libmv_CameraIntrinsics *) new_intrinsics;
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_CameraIntrinsicsDestroy(struct libmv_CameraIntrinsics *libmvIntrinsics)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *intrinsics = (libmv::CameraIntrinsics *) libmvIntrinsics;
|
|
|
|
|
|
|
|
delete intrinsics;
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
void libmv_CameraIntrinsicsUpdate(struct libmv_CameraIntrinsics *libmv_intrinsics,
|
|
|
|
libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
2012-12-10 16:38:28 +00:00
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
double focal_length = libmv_camera_intrinsics_options->focal_length;
|
|
|
|
double principal_x = libmv_camera_intrinsics_options->principal_point_x;
|
|
|
|
double principal_y = libmv_camera_intrinsics_options->principal_point_y;
|
|
|
|
double k1 = libmv_camera_intrinsics_options->k1;
|
|
|
|
double k2 = libmv_camera_intrinsics_options->k2;
|
|
|
|
double k3 = libmv_camera_intrinsics_options->k3;
|
|
|
|
int image_width = libmv_camera_intrinsics_options->image_width;
|
|
|
|
int image_height = libmv_camera_intrinsics_options->image_height;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
/* try avoid unnecessary updates so pre-computed distortion grids are not freed */
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
if (camera_intrinsics->focal_length() != focal_length)
|
|
|
|
camera_intrinsics->SetFocalLength(focal_length, focal_length);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
if (camera_intrinsics->principal_point_x() != principal_x ||
|
|
|
|
camera_intrinsics->principal_point_y() != principal_y)
|
|
|
|
{
|
|
|
|
camera_intrinsics->SetPrincipalPoint(principal_x, principal_y);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (camera_intrinsics->k1() != k1 ||
|
|
|
|
camera_intrinsics->k2() != k2 ||
|
|
|
|
camera_intrinsics->k3() != k3)
|
|
|
|
{
|
|
|
|
camera_intrinsics->SetRadialDistortion(k1, k2, k3);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (camera_intrinsics->image_width() != image_width ||
|
|
|
|
camera_intrinsics->image_height() != image_height)
|
|
|
|
{
|
|
|
|
camera_intrinsics->SetImageSize(image_width, image_height);
|
|
|
|
}
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
|
2013-03-15 11:59:46 +00:00
|
|
|
void libmv_CameraIntrinsicsSetThreads(struct libmv_CameraIntrinsics *libmv_intrinsics, int threads)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
|
|
|
|
|
|
|
|
camera_intrinsics->SetThreads(threads);
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
void libmv_CameraIntrinsicsExtract(struct libmv_CameraIntrinsics *libmv_intrinsics, double *focal_length,
|
|
|
|
double *principal_x, double *principal_y, double *k1, double *k2, double *k3, int *width, int *height)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
|
|
|
|
|
|
|
|
*focal_length = camera_intrinsics->focal_length();
|
|
|
|
*principal_x = camera_intrinsics->principal_point_x();
|
|
|
|
*principal_y = camera_intrinsics->principal_point_y();
|
|
|
|
*k1 = camera_intrinsics->k1();
|
|
|
|
*k2 = camera_intrinsics->k2();
|
Assorted camera tracker improvements
- Add support for refining the camera's intrinsic parameters
during a solve. Currently, refining supports only the following
combinations of intrinsic parameters:
f
f, cx, cy
f, cx, cy, k1, k2
f, k1
f, k1, k2
This is not the same as autocalibration, since the user must
still make a reasonable initial guess about the focal length and
other parameters, whereas true autocalibration would eliminate
the need for the user specify intrinsic parameters at all.
However, the solver works well with only rough guesses for the
focal length, so perhaps full autocalibation is not that
important.
Adding support for the last two combinations, (f, k1) and (f,
k1, k2) required changes to the library libmv depends on for
bundle adjustment, SSBA. These changes should get ported
upstream not just to libmv but to SSBA as well.
- Improved the region of convergence for bundle adjustment by
increasing the number of Levenberg-Marquardt iterations from 50
to 500. This way, the solver is able to crawl out of the bad
local minima it gets stuck in when changing from, for example,
bundling k1 and k2 to just k1 and resetting k2 to 0.
- Add several new region tracker implementations. A region tracker
is a libmv concept, which refers to tracking a template image
pattern through frames. The impact to end users is that tracking
should "just work better". I am reserving a more detailed
writeup, and maybe a paper, for later.
- Other libmv tweaks, such as detecting that a tracker is headed
outside of the image bounds.
This includes several changes made directly to the libmv extern
code rather expecting to get those changes through normal libmv
channels, because I, the libmv BDFL, decided it was faster to work
on libmv directly in Blender, then later reverse-port the libmv
changes from Blender back into libmv trunk. The interesting part
is that I added a full Levenberg-Marquardt loop to the region
tracking code, which should lead to a more stable solutions. I
also added a hacky implementation of "Efficient Second-Order
Minimization" for tracking, which works nicely. A more detailed
quantitative evaluation will follow.
Original patch by Keir, cleaned a bit by myself.
2011-11-14 06:41:23 +00:00
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
void libmv_CameraIntrinsicsUndistortByte(struct libmv_CameraIntrinsics *libmv_intrinsics,
|
2011-11-07 12:55:18 +00:00
|
|
|
unsigned char *src, unsigned char *dst, int width, int height, float overscan, int channels)
|
|
|
|
{
|
2012-12-10 16:38:28 +00:00
|
|
|
libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
camera_intrinsics->Undistort(src, dst, width, height, overscan, channels);
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_CameraIntrinsicsUndistortFloat(struct libmv_CameraIntrinsics *libmvIntrinsics,
|
|
|
|
float *src, float *dst, int width, int height, float overscan, int channels)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *intrinsics = (libmv::CameraIntrinsics *) libmvIntrinsics;
|
|
|
|
|
|
|
|
intrinsics->Undistort(src, dst, width, height, overscan, channels);
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_CameraIntrinsicsDistortByte(struct libmv_CameraIntrinsics *libmvIntrinsics,
|
|
|
|
unsigned char *src, unsigned char *dst, int width, int height, float overscan, int channels)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *intrinsics = (libmv::CameraIntrinsics *) libmvIntrinsics;
|
|
|
|
intrinsics->Distort(src, dst, width, height, overscan, channels);
|
|
|
|
}
|
|
|
|
|
|
|
|
void libmv_CameraIntrinsicsDistortFloat(struct libmv_CameraIntrinsics *libmvIntrinsics,
|
|
|
|
float *src, float *dst, int width, int height, float overscan, int channels)
|
|
|
|
{
|
|
|
|
libmv::CameraIntrinsics *intrinsics = (libmv::CameraIntrinsics *) libmvIntrinsics;
|
|
|
|
|
|
|
|
intrinsics->Distort(src, dst, width, height, overscan, channels);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ************ utils ************ */
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
void libmv_applyCameraIntrinsics(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options,
|
|
|
|
double x, double y, double *x1, double *y1)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
2012-12-10 16:38:28 +00:00
|
|
|
libmv::CameraIntrinsics camera_intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
cameraIntrinsicsFromOptions(&camera_intrinsics, libmv_camera_intrinsics_options);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
if (libmv_camera_intrinsics_options->focal_length) {
|
2011-11-07 12:55:18 +00:00
|
|
|
/* do a lens undistortion if focal length is non-zero only */
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
camera_intrinsics.ApplyIntrinsics(x, y, x1, y1);
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
void libmv_InvertIntrinsics(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options,
|
|
|
|
double x, double y, double *x1, double *y1)
|
2011-11-07 12:55:18 +00:00
|
|
|
{
|
2012-12-10 16:38:28 +00:00
|
|
|
libmv::CameraIntrinsics camera_intrinsics;
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
cameraIntrinsicsFromOptions(&camera_intrinsics, libmv_camera_intrinsics_options);
|
2011-11-07 12:55:18 +00:00
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
if (libmv_camera_intrinsics_options->focal_length) {
|
2011-11-07 12:55:18 +00:00
|
|
|
/* do a lens distortion if focal length is non-zero only */
|
|
|
|
|
2012-12-10 16:38:28 +00:00
|
|
|
camera_intrinsics.InvertIntrinsics(x, y, x1, y1);
|
2011-11-07 12:55:18 +00:00
|
|
|
}
|
|
|
|
}
|