Camera tracking integration
=========================== Some improvements for feature detectors: - Sort features by score when filtering features by distance using FAST library. - Added option to place markers only in areas outlined by grease pencil.
This commit is contained in:
Sergey Sharybin
parent
e168a4f019
commit
8fb37629cd
42
extern/libmv/libmv/simple_pipeline/detect.cc
vendored
42
extern/libmv/libmv/simple_pipeline/detect.cc
vendored
@ -35,6 +35,14 @@ namespace libmv {
|
||||
|
||||
typedef unsigned int uint;
|
||||
|
||||
int featurecmp(const void *a_v, const void *b_v)
|
||||
{
|
||||
Feature *a = (Feature*)a_v;
|
||||
Feature *b = (Feature*)b_v;
|
||||
|
||||
return b->score - a->score;
|
||||
}
|
||||
|
||||
std::vector<Feature> DetectFAST(const unsigned char* data, int width, int height, int stride,
|
||||
int min_trackness, int min_distance) {
|
||||
std::vector<Feature> features;
|
||||
@ -54,21 +62,43 @@ std::vector<Feature> DetectFAST(const unsigned char* data, int width, int height
|
||||
// TODO(MatthiasF): A resolution independent parameter would be better than distance
|
||||
// e.g. a coefficient going from 0 (no minimal distance) to 1 (optimal circle packing)
|
||||
// FIXME(MatthiasF): this method will not necessarily give all maximum markers
|
||||
if(num_features) features.reserve(num_features);
|
||||
if(num_features) {
|
||||
Feature *all_features = new Feature[num_features];
|
||||
|
||||
for(int i = 0; i < num_features; ++i) {
|
||||
xy xy = nonmax[i];
|
||||
Feature a = { xy.x, xy.y, scores[i], 7 };
|
||||
Feature a = { nonmax[i].x, nonmax[i].y, scores[i], 0 };
|
||||
all_features[i] = a;
|
||||
}
|
||||
|
||||
qsort((void *)all_features, num_features, sizeof(Feature), featurecmp);
|
||||
|
||||
features.reserve(num_features);
|
||||
|
||||
int prev_score = all_features[0].score;
|
||||
for(int i = 0; i < num_features; ++i) {
|
||||
bool ok = true;
|
||||
Feature a = all_features[i];
|
||||
if(a.score>prev_score)
|
||||
abort();
|
||||
prev_score = a.score;
|
||||
|
||||
// compare each feature against filtered set
|
||||
for(int j = 0; j < features.size(); j++) {
|
||||
Feature& b = features[j];
|
||||
if ( (a.x-b.x)*(a.x-b.x)+(a.y-b.y)*(a.y-b.y) < min_distance*min_distance ) {
|
||||
// already a nearby feature
|
||||
goto skip;
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// otherwise add the new feature
|
||||
|
||||
if(ok) {
|
||||
// add the new feature
|
||||
features.push_back(a);
|
||||
skip: ;
|
||||
}
|
||||
}
|
||||
|
||||
delete [] all_features;
|
||||
}
|
||||
free(scores);
|
||||
free(nonmax);
|
||||
|
||||
48
extern/libmv/patches/detect.patch
vendored
48
extern/libmv/patches/detect.patch
vendored
@ -1,8 +1,8 @@
|
||||
diff --git a/src/libmv/simple_pipeline/detect.cc b/src/libmv/simple_pipeline/detect.cc
|
||||
index 6fc0cdd..b316f42 100644
|
||||
index 6fc0cdd..8ac42ab 100644
|
||||
--- a/src/libmv/simple_pipeline/detect.cc
|
||||
+++ b/src/libmv/simple_pipeline/detect.cc
|
||||
@@ -23,15 +23,59 @@
|
||||
@@ -23,15 +23,89 @@
|
||||
****************************************************************************/
|
||||
|
||||
#include "libmv/simple_pipeline/detect.h"
|
||||
@ -19,6 +19,14 @@ index 6fc0cdd..b316f42 100644
|
||||
|
||||
typedef unsigned int uint;
|
||||
|
||||
+int featurecmp(const void *a_v, const void *b_v)
|
||||
+{
|
||||
+ Feature *a = (Feature*)a_v;
|
||||
+ Feature *b = (Feature*)b_v;
|
||||
+
|
||||
+ return b->score - a->score;
|
||||
+}
|
||||
+
|
||||
+std::vector<Feature> DetectFAST(const unsigned char* data, int width, int height, int stride,
|
||||
+ int min_trackness, int min_distance) {
|
||||
+ std::vector<Feature> features;
|
||||
@ -38,21 +46,43 @@ index 6fc0cdd..b316f42 100644
|
||||
+ // TODO(MatthiasF): A resolution independent parameter would be better than distance
|
||||
+ // e.g. a coefficient going from 0 (no minimal distance) to 1 (optimal circle packing)
|
||||
+ // FIXME(MatthiasF): this method will not necessarily give all maximum markers
|
||||
+ if(num_features) features.reserve(num_features);
|
||||
+ if(num_features) {
|
||||
+ Feature *all_features = new Feature[num_features];
|
||||
+
|
||||
+ for(int i = 0; i < num_features; ++i) {
|
||||
+ xy xy = nonmax[i];
|
||||
+ Feature a = { xy.x, xy.y, scores[i], 7 };
|
||||
+ Feature a = { nonmax[i].x, nonmax[i].y, scores[i], 0 };
|
||||
+ all_features[i] = a;
|
||||
+ }
|
||||
+
|
||||
+ qsort((void *)all_features, num_features, sizeof(Feature), featurecmp);
|
||||
+
|
||||
+ features.reserve(num_features);
|
||||
+
|
||||
+ int prev_score = all_features[0].score;
|
||||
+ for(int i = 0; i < num_features; ++i) {
|
||||
+ bool ok = true;
|
||||
+ Feature a = all_features[i];
|
||||
+ if(a.score>prev_score)
|
||||
+ abort();
|
||||
+ prev_score = a.score;
|
||||
+
|
||||
+ // compare each feature against filtered set
|
||||
+ for(int j = 0; j < features.size(); j++) {
|
||||
+ Feature& b = features[j];
|
||||
+ if ( (a.x-b.x)*(a.x-b.x)+(a.y-b.y)*(a.y-b.y) < min_distance*min_distance ) {
|
||||
+ // already a nearby feature
|
||||
+ goto skip;
|
||||
+ ok = false;
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+ // otherwise add the new feature
|
||||
+
|
||||
+ if(ok) {
|
||||
+ // add the new feature
|
||||
+ features.push_back(a);
|
||||
+ skip: ;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ delete [] all_features;
|
||||
+ }
|
||||
+ free(scores);
|
||||
+ free(nonmax);
|
||||
@ -64,7 +94,7 @@ index 6fc0cdd..b316f42 100644
|
||||
static uint SAD(const ubyte* imageA, const ubyte* imageB, int strideA, int strideB) {
|
||||
__m128i a = _mm_setzero_si128();
|
||||
for(int i = 0; i < 16; i++) {
|
||||
@@ -52,7 +96,7 @@ static uint SAD(const ubyte* imageA, const ubyte* imageB, int strideA, int strid
|
||||
@@ -52,7 +126,7 @@ static uint SAD(const ubyte* imageA, const ubyte* imageB, int strideA, int strid
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -34,6 +34,7 @@
|
||||
* \author Sergey Sharybin
|
||||
*/
|
||||
|
||||
struct bGPDlayer;
|
||||
struct ImBuf;
|
||||
struct MovieTrackingTrack;
|
||||
struct MovieTrackingMarker;
|
||||
@ -87,7 +88,11 @@ void BKE_tracking_projection_matrix(struct MovieTracking *tracking, int framenr,
|
||||
void BKE_tracking_apply_intrinsics(struct MovieTracking *tracking, float co[2], float nco[2]);
|
||||
void BKE_tracking_invert_intrinsics(struct MovieTracking *tracking, float co[2], float nco[2]);
|
||||
|
||||
void BKE_tracking_detect(struct MovieTracking *tracking, struct ImBuf *imbuf, int framenr, int margin, int min_trackness, int count, int min_distance, int fast);
|
||||
void BKE_tracking_detect_fast(struct MovieTracking *tracking, struct ImBuf *imbuf,
|
||||
int framenr, int margin, int min_trackness, int min_distance, struct bGPDlayer *layer);
|
||||
|
||||
void BKE_tracking_detect_moravec(struct MovieTracking *tracking, struct ImBuf *imbuf,
|
||||
int framenr, int margin, int count, int min_distance, struct bGPDlayer *layer);
|
||||
|
||||
struct MovieTrackingTrack *BKE_tracking_indexed_bundle(struct MovieTracking *tracking, int bundlenr);
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
#include "DNA_gpencil_types.h"
|
||||
#include "DNA_movieclip_types.h"
|
||||
#include "DNA_object_types.h" /* SELECT */
|
||||
#include "DNA_scene_types.h"
|
||||
@ -1495,7 +1496,6 @@ void BKE_tracking_invert_intrinsics(MovieTracking *tracking, float co[2], float
|
||||
}
|
||||
|
||||
#ifdef WITH_LIBMV
|
||||
/* flips upside-down */
|
||||
static unsigned char *acquire_ucharbuf(ImBuf *ibuf)
|
||||
{
|
||||
int x, y;
|
||||
@ -1504,18 +1504,16 @@ static unsigned char *acquire_ucharbuf(ImBuf *ibuf)
|
||||
fp= pixels= MEM_callocN(ibuf->x*ibuf->y*sizeof(unsigned char), "tracking ucharBuf");
|
||||
for(y= 0; y<ibuf->y; y++) {
|
||||
for (x= 0; x<ibuf->x; x++) {
|
||||
int pixel= ibuf->x*(ibuf->y-y-1) + x;
|
||||
int pixel= ibuf->x*y + x;
|
||||
|
||||
if(ibuf->rect_float) {
|
||||
float *rrgbf= ibuf->rect_float + pixel*4;
|
||||
|
||||
//*fp= 0.2126f*rrgbf[0] + 0.7152f*rrgbf[1] + 0.0722f*rrgbf[2];
|
||||
*fp= (11*rrgbf[0]+16*rrgbf[1]+5*rrgbf[2])/32;
|
||||
*fp= 0.2126f*rrgbf[0] + 0.7152f*rrgbf[1] + 0.0722f*rrgbf[2];
|
||||
} else {
|
||||
char *rrgb= (char*)ibuf->rect + pixel*4;
|
||||
|
||||
//*fp= 0.2126f*rrgb[0] + 0.7152f*rrgb[1] + 0.0722f*rrgb[2];
|
||||
*fp= (11*rrgb[0]+16*rrgb[1]+5*rrgb[2])/32;
|
||||
*fp= 0.2126f*rrgb[0] + 0.7152f*rrgb[1] + 0.0722f*rrgb[2];
|
||||
}
|
||||
|
||||
fp++;
|
||||
@ -1526,32 +1524,106 @@ static unsigned char *acquire_ucharbuf(ImBuf *ibuf)
|
||||
}
|
||||
#endif
|
||||
|
||||
void BKE_tracking_detect(MovieTracking *tracking, ImBuf *ibuf, int framenr, int margin, int min_trackness, int count, int min_distance, int fast)
|
||||
static int point_in_stroke(bGPDstroke *stroke, float x, float y)
|
||||
{
|
||||
int i, prev;
|
||||
int count= 0;
|
||||
bGPDspoint *points= stroke->points;
|
||||
|
||||
prev= stroke->totpoints-1;
|
||||
|
||||
for(i= 0; i<stroke->totpoints; i++) {
|
||||
if((points[i].y<y && points[prev].y>=y) || (points[prev].y<y && points[i].y>=y)) {
|
||||
float fac= (y-points[i].y)/(points[prev].y-points[i].y);
|
||||
|
||||
if (points[i].x+fac*(points[prev].x-points[i].x)<x)
|
||||
count++;
|
||||
}
|
||||
|
||||
prev= i;
|
||||
}
|
||||
|
||||
return count%2;
|
||||
}
|
||||
|
||||
static int point_in_layer(bGPDlayer *layer, float x, float y)
|
||||
{
|
||||
bGPDframe *frame= layer->frames.first;
|
||||
|
||||
while(frame) {
|
||||
bGPDstroke *stroke= frame->strokes.first;
|
||||
while(stroke) {
|
||||
if(point_in_stroke(stroke, x, y))
|
||||
return 1;
|
||||
|
||||
stroke= stroke->next;
|
||||
}
|
||||
frame= frame->next;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void retrive_libmv_features(MovieTracking *tracking, struct libmv_Features *features,
|
||||
int framenr, int width, int height, bGPDlayer *layer)
|
||||
{
|
||||
#ifdef WITH_LIBMV
|
||||
struct libmv_Features *features;
|
||||
unsigned char *pixels= acquire_ucharbuf(ibuf);
|
||||
int a;
|
||||
|
||||
if(fast)
|
||||
features= libmv_detectFeaturesFAST(pixels, ibuf->x, ibuf->y, ibuf->x, margin, min_trackness, min_distance);
|
||||
else
|
||||
features= libmv_detectFeaturesMORAVEC(pixels, ibuf->x, ibuf->y, ibuf->x, margin, count, min_distance);
|
||||
|
||||
MEM_freeN(pixels);
|
||||
|
||||
a= libmv_countFeatures(features);
|
||||
while(a--) {
|
||||
MovieTrackingTrack *track;
|
||||
double x, y, size, score;
|
||||
int ok= 1;
|
||||
float xu, yu;
|
||||
|
||||
libmv_getFeature(features, a, &x, &y, &score, &size);
|
||||
|
||||
track= BKE_tracking_add_track(tracking, x/ibuf->x, 1.0f-(y/ibuf->y), framenr, ibuf->x, ibuf->y);
|
||||
xu= x/width;
|
||||
yu= y/height;
|
||||
|
||||
if(layer)
|
||||
ok= point_in_layer(layer, xu, yu);
|
||||
|
||||
if(ok) {
|
||||
track= BKE_tracking_add_track(tracking, xu, yu, framenr, width, height);
|
||||
track->flag|= SELECT;
|
||||
track->pat_flag|= SELECT;
|
||||
track->search_flag|= SELECT;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void BKE_tracking_detect_fast(MovieTracking *tracking, ImBuf *ibuf,
|
||||
int framenr, int margin, int min_trackness, int min_distance, bGPDlayer *layer)
|
||||
{
|
||||
#ifdef WITH_LIBMV
|
||||
struct libmv_Features *features;
|
||||
unsigned char *pixels= acquire_ucharbuf(ibuf);
|
||||
|
||||
features= libmv_detectFeaturesFAST(pixels, ibuf->x, ibuf->y, ibuf->x, margin, min_trackness, min_distance);
|
||||
|
||||
MEM_freeN(pixels);
|
||||
|
||||
retrive_libmv_features(tracking, features, framenr, ibuf->x, ibuf->y, layer);
|
||||
|
||||
libmv_destroyFeatures(features);
|
||||
#endif
|
||||
}
|
||||
|
||||
void BKE_tracking_detect_moravec(MovieTracking *tracking, ImBuf *ibuf,
|
||||
int framenr, int margin, int count, int min_distance, bGPDlayer *layer)
|
||||
{
|
||||
#ifdef WITH_LIBMV
|
||||
struct libmv_Features *features;
|
||||
unsigned char *pixels= acquire_ucharbuf(ibuf);
|
||||
|
||||
features= libmv_detectFeaturesMORAVEC(pixels, ibuf->x, ibuf->y, ibuf->x, margin, count, min_distance);
|
||||
|
||||
MEM_freeN(pixels);
|
||||
|
||||
retrive_libmv_features(tracking, features, framenr, ibuf->x, ibuf->y, layer);
|
||||
|
||||
libmv_destroyFeatures(features);
|
||||
#endif
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
#include "DNA_camera_types.h"
|
||||
#include "DNA_gpencil_types.h"
|
||||
#include "DNA_movieclip_types.h"
|
||||
#include "DNA_object_types.h" /* SELECT */
|
||||
#include "DNA_scene_types.h"
|
||||
@ -2208,6 +2209,24 @@ void CLIP_OT_hide_tracks_clear(wmOperatorType *ot)
|
||||
|
||||
/********************** detect features operator *********************/
|
||||
|
||||
static bGPDlayer *detect_get_layer(MovieClip *clip)
|
||||
{
|
||||
bGPDlayer *layer;
|
||||
|
||||
if(!clip->gpd)
|
||||
return NULL;
|
||||
|
||||
layer= clip->gpd->layers.first;
|
||||
while(layer) {
|
||||
if(layer->flag&GP_LAYER_ACTIVE)
|
||||
return layer;
|
||||
|
||||
layer= layer->next;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int detect_features_exec(bContext *C, wmOperator *op)
|
||||
{
|
||||
SpaceClip *sc= CTX_wm_space_clip(C);
|
||||
@ -2215,10 +2234,15 @@ static int detect_features_exec(bContext *C, wmOperator *op)
|
||||
ImBuf *ibuf= BKE_movieclip_acquire_ibuf_flag(clip, &sc->user, 0);
|
||||
MovieTrackingTrack *track= clip->tracking.tracks.first;
|
||||
int detector= RNA_enum_get(op->ptr, "detector");
|
||||
int use_grease_pencil= RNA_boolean_get(op->ptr, "use_grease_pencil");
|
||||
int margin= RNA_int_get(op->ptr, "margin");
|
||||
int min_trackness= RNA_int_get(op->ptr, "min_trackness");
|
||||
int count= RNA_int_get(op->ptr, "count");
|
||||
int min_distance= RNA_int_get(op->ptr, "min_distance");
|
||||
bGPDlayer *layer= NULL;
|
||||
|
||||
if(use_grease_pencil)
|
||||
layer= detect_get_layer(clip);
|
||||
|
||||
/* deselect existing tracks */
|
||||
while(track) {
|
||||
@ -2229,7 +2253,10 @@ static int detect_features_exec(bContext *C, wmOperator *op)
|
||||
track= track->next;
|
||||
}
|
||||
|
||||
BKE_tracking_detect(&clip->tracking, ibuf, sc->user.framenr, margin, min_trackness, count, min_distance, detector==0);
|
||||
if(detector==0)
|
||||
BKE_tracking_detect_fast(&clip->tracking, ibuf, sc->user.framenr, margin, min_trackness, min_distance, layer);
|
||||
else
|
||||
BKE_tracking_detect_moravec(&clip->tracking, ibuf, sc->user.framenr, margin, count, min_distance, layer);
|
||||
|
||||
IMB_freeImBuf(ibuf);
|
||||
|
||||
@ -2287,6 +2314,7 @@ void CLIP_OT_detect_features(wmOperatorType *ot)
|
||||
|
||||
/* properties */
|
||||
RNA_def_enum(ot->srna, "detector", detector_items, 0, "Detector", "Detector using for detecting features");
|
||||
RNA_def_boolean(ot->srna, "use_grease_pencil", 0, "Use Grease Pencil", "Use grease pencil strokes from active layer to define zones where detection should happen");
|
||||
RNA_def_int(ot->srna, "margin", 16, 0, INT_MAX, "Margin", "Only corners further than margin pixels from the image edges are considered", 0, 300);
|
||||
RNA_def_int(ot->srna, "min_trackness", 16, 0, INT_MAX, "Trackness", "Minimum score to add a corner", 0, 300);
|
||||
RNA_def_int(ot->srna, "count", 50, 1, INT_MAX, "Count", "Count of corners to detect", 0, 300);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user