From 2309cc4fbbdb478bb25c5f030eba35b73850b52a Mon Sep 17 00:00:00 2001 From: Paolo Cignoni cignoni Date: Fri, 2 Dec 2011 14:28:54 +0000 Subject: [PATCH] Optical flow experimental filter modified so as to include borders defined by weight ratio to each dominant region. --- .../DominancyClassifier.cpp | 111 +++++----- .../DominancyClassifier.h | 42 ++-- .../filter_output_optical_flow.cpp | 189 +++++++++--------- .../filter_output_optical_flow.h | 8 +- 4 files changed, 169 insertions(+), 181 deletions(-) diff --git a/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.cpp b/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.cpp index a80e5a221..73b31727f 100644 --- a/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.cpp +++ b/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.cpp @@ -44,7 +44,7 @@ DominancyClassifier::DominancyClassifier( CMeshO &mesh, QList &raster { projectiveTexMatrices( *r ); setupShadowTexture( *r ); - generateWeightsAndShadowMap(); + generateWeightsAndShadowMap( *r ); checkDominancy( *r ); } @@ -216,7 +216,7 @@ void DominancyClassifier::setupShadowTexture( OOCRaster &rr ) } -void DominancyClassifier::generateWeightsAndShadowMap() +void DominancyClassifier::generateWeightsAndShadowMap( OOCRaster &rr ) { // Backup the previous OpenGL states. glPushAttrib( GL_VIEWPORT_BIT | @@ -296,6 +296,20 @@ delete [] b; buffer2.init( fbuffer.Width(), fbuffer.Height() ); fbuffer.DumpTo( GL_DEPTH_ATTACHMENT, buffer1.data, GL_DEPTH_COMPONENT, GL_FLOAT ); // WARNING: leads to a GL error. + + float zNear, zFar; + GlShot< vcg::Shot >::GetNearFarPlanes( rr.shot, m_Mesh.bbox, zNear, zFar ); + if( zNear < 0.0001f ) + zNear = 0.1f; + if( zFar < zNear ) + zFar = zNear + 1000.0f; + + float range = zFar - zNear; + for( unsigned int i=0; i m_VertexDom[v].weight ) + if( weightBuffer[v] > m_VertexDom[v].weight1 ) { - m_VertexDom[v].weight = weightBuffer[v]; - m_VertexDom[v].dominant = &rr; + m_VertexDom[v].weight2 = m_VertexDom[v].weight1; + m_VertexDom[v].dominant2 = m_VertexDom[v].dominant1; + + m_VertexDom[v].weight1 = weightBuffer[v]; + m_VertexDom[v].dominant1 = &rr; + } + else if( weightBuffer[v] > m_VertexDom[v].weight2 ) + { + m_VertexDom[v].weight2 = weightBuffer[v]; + m_VertexDom[v].dominant2 = &rr; } delete [] weightBuffer; @@ -403,62 +425,33 @@ void DominancyClassifier::releaseAll() } -void DominancyClassifier::facesWithDominant( FaceVec &faces, - const OOCRaster *rr, - FaceDomMode mode ) const +void DominancyClassifier::dominancyCoverage( RasterFaceMap &rpatches ) const { - faces.clear(); + rpatches.clear(); - if( mode == FD_AGRESSIVE ) + for( CMeshO::FaceIterator f=m_Mesh.face.begin(); f!=m_Mesh.face.end(); ++f ) { - for( CMeshO::FaceIterator f=m_Mesh.face.begin(); f!=m_Mesh.face.end(); ++f ) - if( (*this)[f->V(0)].dominant == rr || - (*this)[f->V(1)].dominant == rr || - (*this)[f->V(2)].dominant == rr ) - faces.push_back( &*f ); - } - else - { - for( CMeshO::FaceIterator f=m_Mesh.face.begin(); f!=m_Mesh.face.end(); ++f ) - if( (*this)[f->V(0)].dominant == rr && - (*this)[f->V(1)].dominant == rr && - (*this)[f->V(2)].dominant == rr ) - faces.push_back( &*f ); - } -} - - -void DominancyClassifier::dominancyCoverage( RasterFaceMap &faces, - FaceDomMode mode ) const -{ - faces.clear(); - - if( mode == FD_AGRESSIVE ) - { - for( CMeshO::FaceIterator f=m_Mesh.face.begin(); f!=m_Mesh.face.end(); ++f ) - { - OOCRaster *d0 = (*this)[f->V(0)].dominant; - OOCRaster *d1 = (*this)[f->V(1)].dominant; - OOCRaster *d2 = (*this)[f->V(2)].dominant; - - if( d0 ) - faces[d0].push_back( &*f ); - if( d1 && d1!=d0 ) - faces[d1].push_back( &*f ); - if( d2 && d2!=d0 && d2!=d1 ) - faces[d2].push_back( &*f ); - } - } - else - { - for( CMeshO::FaceIterator f=m_Mesh.face.begin(); f!=m_Mesh.face.end(); ++f ) - { - OOCRaster *d0 = (*this)[f->V(0)].dominant; - OOCRaster *d1 = (*this)[f->V(1)].dominant; - OOCRaster *d2 = (*this)[f->V(2)].dominant; - - if( d0 && d0==d1 && d0==d2 ) - faces[d0].push_back( &*f ); - } + const VDominancy &d0 = (*this)[f->V(0)]; + const VDominancy &d1 = (*this)[f->V(1)]; + const VDominancy &d2 = (*this)[f->V(2)]; + + std::set rastersFBelongsTo; + + if( d0.dominant1 ) + rastersFBelongsTo.insert( d0.dominant1 ); + if( d1.dominant1 ) + rastersFBelongsTo.insert( d1.dominant1 ); + if( d2.dominant1 ) + rastersFBelongsTo.insert( d2.dominant1 ); + + if( d0.isOnBoundary() ) + rastersFBelongsTo.insert( d0.dominant2 ); + if( d1.isOnBoundary() ) + rastersFBelongsTo.insert( d1.dominant2 ); + if( d2.isOnBoundary() ) + rastersFBelongsTo.insert( d2.dominant2 ); + + for( std::set::iterator r=rastersFBelongsTo.begin(); r!=rastersFBelongsTo.end(); ++r ) + rpatches[*r].push_back( &*f ); } } diff --git a/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.h b/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.h index 2db32224f..e162cec66 100644 --- a/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.h +++ b/src/plugins_experimental/filter_ouput_optical_flow/DominancyClassifier.h @@ -33,6 +33,16 @@ typedef QVector FaceVec; +typedef QVector WeightVec; + +struct Patch +{ + OOCRaster *ref; + FaceVec faces; + FaceVec boundary; + WeightVec bWeight; +}; + typedef QMap RasterFaceMap; @@ -48,21 +58,22 @@ public: W_ALL = 0x0F, }; - enum FaceDomMode - { - FD_STRICT , - FD_AGRESSIVE , - }; - struct VDominancy { - float weight; - OOCRaster *dominant; + float weight1; + float weight2; + OOCRaster *dominant1; + OOCRaster *dominant2; inline VDominancy() : - weight(-1.0f), - dominant(NULL) + weight1(-1.0f), + weight2(-1.0f), + dominant1(NULL), + dominant2(NULL) {} + + inline bool isOnBoundary() const { return dominant1 && dominant2 && borderWeight()<=1.0f; } + inline float borderWeight() const { return (std::sqrt(weight1/weight2) - 1.0f) / 0.2; } }; private: @@ -101,13 +112,13 @@ private: void updateDepthRange(); void updateMeshVBO(); bool initShaders(); - void generateWeightsAndShadowMap(); + void generateWeightsAndShadowMap( OOCRaster &rr ); void projectiveTexMatrices( OOCRaster &rr ); void setupShadowTexture( OOCRaster &rr ); void checkDominancy( OOCRaster &rr ); void releaseAll(); - inline int id( const CVertexO& v ) const { return &v - &m_Mesh.vert[0]; } + inline int id( const CVertexO& v ) const { return &v - &m_Mesh.vert[0]; } public: DominancyClassifier( CMeshO &mesh, QList &rasterList, int weightMask ); @@ -121,12 +132,7 @@ public: inline const VDominancy& operator[]( const CMeshO::VertexIterator& v ) const { return m_VertexDom[id(*v)]; } inline VDominancy& operator[]( const CMeshO::VertexIterator& v ) { return m_VertexDom[id(*v)]; } - void facesWithDominant( FaceVec &faces, - const OOCRaster *rr, - FaceDomMode mode = FD_AGRESSIVE ) const; - - void dominancyCoverage( RasterFaceMap &faces, - FaceDomMode mode = FD_AGRESSIVE ) const; + void dominancyCoverage( RasterFaceMap &rpatches ) const; }; diff --git a/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.cpp b/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.cpp index 65ccefeb0..6775998b6 100644 --- a/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.cpp +++ b/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.cpp @@ -109,10 +109,10 @@ void FilterOutputOpticalFlowPlugin::initParameterSet( QAction *act, true, "Use orientation weight", "Includes a weight accounting for the orientation of the surface wrt. the camera during the computation of reference images") ); - par.addParam( new RichInt("dominantAreaExpansion", - 4, - "Area expansion", - "Width (in triangles) of the border to add to each dominant area.") ); + //par.addParam( new RichInt("dominantAreaExpansion", + // 4, + // "Area expansion", + // "Width (in triangles) of the border to add to each dominant area.") ); par.addParam( new RichFloat("minCoverage", 2.0f, "Min coverage (%)", @@ -167,10 +167,7 @@ bool FilterOutputOpticalFlowPlugin::applyFilter( QAction *act, case FP_OUTPUT_OPTICAL_FLOW: { for( CMeshO::FaceIterator f=m_Mesh->face.begin(); f!=m_Mesh->face.end(); ++f ) - { f->ClearV(); - f->ClearS(); - } int weightMask = 0; if( par.getBool("useDistanceWeight") ) @@ -182,48 +179,43 @@ bool FilterOutputOpticalFlowPlugin::applyFilter( QAction *act, if( par.getBool("useOrientationWeight") ) weightMask |= DominancyClassifier::W_ORIENTATION; - RasterFaceMap facesByDomImg; DominancyClassifier *set = new DominancyClassifier( *m_Mesh, rasters, weightMask ); if( par.getBool("colorFromDominancy") ) { md.mm()->updateDataMask( MeshModel::MM_VERTCOLOR ); - QMap rasterCol; - for( QList::iterator r=rasters.begin(); r!=rasters.end(); ++r ) - rasterCol[&*r] = vcg::Color4b( (rand()&127)+128, (rand()&127)+128, (rand()&127)+128, 255 ); - for( CMeshO::VertexIterator v=m_Mesh->vert.begin(); v!=m_Mesh->vert.end(); ++v ) - { - OOCRaster *dom = (*set)[v].dominant; - if( dom ) - v->C() = rasterCol[dom]; + if( (*set)[v].isOnBoundary() ) + { + unsigned char c = (unsigned char)( 255.0f*(*set)[v].borderWeight() ); + v->C() = vcg::Color4b( 255-c, 255-c, 255-c, 255 ); + } else - v->C() = vcg::Color4b( 0, 0, 0, 255 ); - } + v->C() = vcg::Color4b( 0, 0, 255, 255 ); delete set; - break; } - - set->dominancyCoverage( facesByDomImg, DominancyClassifier::FD_AGRESSIVE ); - delete set; - - - int n = par.getInt("dominantAreaExpansion"); - if( n ) + else { - md.mm()->updateDataMask( MeshModel::MM_FACEFACETOPO ); - md.mm()->updateDataMask( MeshModel::MM_VERTFACETOPO ); + RasterFaceMap facesByDomImg; + set->dominancyCoverage( facesByDomImg ); + delete set; - for( RasterFaceMap::iterator rm=facesByDomImg.begin(); rm!=facesByDomImg.end(); ++rm ) - expands( rm.value(), n ); + //if( int n = par.getInt("dominantAreaExpansion") ) + //{ + // md.mm()->updateDataMask( MeshModel::MM_FACEFACETOPO ); + // md.mm()->updateDataMask( MeshModel::MM_VERTFACETOPO ); + + // for( RasterFaceMap::iterator rm=facesByDomImg.begin(); rm!=facesByDomImg.end(); ++rm ) + // expands( rm.value(), n ); + //} + + QMap> validPairs; + retroProjection( facesByDomImg, 0.01f*par.getFloat("minCoverage"), validPairs ); + saveXMLProject( par.getString("xmlFileName"), md.mm(), facesByDomImg, validPairs ); } - QMap> validPairs; - retroProjection( facesByDomImg, 0.01f*par.getFloat("minCoverage"), validPairs ); - saveXMLProject( par.getString("xmlFileName"), md.mm(), facesByDomImg, validPairs ); - break; } } @@ -300,7 +292,7 @@ bool FilterOutputOpticalFlowPlugin::loadRasterList( QString &mlpFilename, void FilterOutputOpticalFlowPlugin::saveXMLProject( const QString &filename, MeshModel *mm, - RasterFaceMap &rasters, + RasterFaceMap &rpatches, QMap> &validPairs ) { std::ofstream xmlFile( filename.toAscii() ); @@ -313,7 +305,7 @@ void FilterOutputOpticalFlowPlugin::saveXMLProject( const QString &filename, << "\" aligned=\"0\" type=\"GlModelWidget\" numPoints=\"0\" id=\"0\"/>" << std::endl; - for( RasterFaceMap::iterator r=rasters.begin(); r!=rasters.end(); ++r ) + for( RasterFaceMap::iterator r=rpatches.begin(); r!=rpatches.end(); ++r ) { vcg::Shotf &shot = r.key()->shot; @@ -386,65 +378,80 @@ void FilterOutputOpticalFlowPlugin::getNeighbors( CFaceO *f, } -void FilterOutputOpticalFlowPlugin::expands( FaceVec &faces, - int n ) +void FilterOutputOpticalFlowPlugin::expands( Patch &patch, + int nbGrows ) { - for( FaceVec::iterator f=faces.begin(); f!=faces.end(); ++f ) + patch.boundary.clear(); + patch.bWeight.clear(); + + + // Mark all faces that belong to the current patch as "VISITED". + for( FaceVec::iterator f=patch.faces.begin(); f!=patch.faces.end(); ++f ) (*f)->SetV(); - FaceVec *boundary = new FaceVec(), *boundaryOld = new FaceVec(); - for( FaceVec::iterator f=faces.begin(); f!=faces.end(); ++f ) - if( ((*f)->FFp(0) && !(*f)->FFp(0)->IsV()) || - ((*f)->FFp(1) && !(*f)->FFp(1)->IsV()) || - ((*f)->FFp(2) && !(*f)->FFp(2)->IsV()) ) - boundaryOld->push_back( *f ); + // Get vertices that belong to the boundary of the patch by checking all triangle edges. + std::set candidates; - - for( int nbGrow=0; nbGrowclear(); - - for( FaceVec::iterator f=boundaryOld->begin(); f!=boundaryOld->end(); ++f ) + vcg::face::Pos pos( *f, (*f)->V(0) ); + for( int i=0; i<3; ++i ) { - vcg::face::Pos pos( *f, (*f)->V(0) ); + if( pos.FFlip() && !pos.FFlip()->IsV() ) + { + candidates.insert( pos.V() ); + candidates.insert( pos.VFlip() ); + } + pos.FlipV(); + pos.FlipE(); + } + } + + + // For each round of region growing... + for( int n=0; n::iterator v=candidates.begin(); v!=candidates.end(); ++v ) + { + NeighbSet neighb; + getNeighbors( *v, neighb ); + for( NeighbSet::iterator nn=neighb.begin(); nn!=neighb.end(); ++nn ) + if( !(*nn)->IsV() ) + { + (*nn)->SetV(); + patch.boundary.push_back( *nn ); + } + } + + // Vertices of the new boundary are recovered from the triangles that have juste been extracted + // in order to prepare the next region growing round. + for( candidates.clear(); k pos( patch.boundary[k], patch.boundary[k]->V(0) ); for( int i=0; i<3; ++i ) { - const CFaceO *f2 = pos.FFlip(); - if( f2 && !f2->IsV() ) + if( pos.FFlip() && !pos.FFlip()->IsV() ) { - NeighbSet neighb; - getNeighbors( pos.V(), neighb ); - getNeighbors( pos.VFlip(), neighb ); - for( NeighbSet::iterator n=neighb.begin(); n!=neighb.end(); ++n ) - if( !(*n)->IsS() && !(*n)->IsV() ) - { - boundary->push_back( *n ); - (*n)->SetS(); - } + candidates.insert( pos.V() ); + candidates.insert( pos.VFlip() ); } pos.FlipV(); pos.FlipE(); } } - - for( FaceVec::iterator f=boundary->begin(); f!=boundary->end(); ++f ) - { - (*f)->SetV(); - (*f)->ClearS(); - faces.push_back( *f ); - } - - FaceVec *tmp = boundary; - boundary = boundaryOld; - boundaryOld = tmp; } - delete boundary; - delete boundaryOld; + // Unmark all marked faces, for other futur processing. + for( FaceVec::iterator f=patch.faces.begin(); f!=patch.faces.end(); ++f ) + (*f)->ClearV(); - for( FaceVec::iterator f=faces.begin(); f!=faces.end(); ++f ) + for( FaceVec::iterator f=patch.boundary.begin(); f!=patch.boundary.end(); ++f ) (*f)->ClearV(); } @@ -492,6 +499,7 @@ void FilterOutputOpticalFlowPlugin::shadowTextureMatrices( OOCRaster *rr, float l, r, b, t, focal; rr->shot.Intrinsics.GetFrustum( l, r, b, t, focal ); + // Compute from the frustum values the camera projection matrix. proj.SetZero(); proj[0][0] = 2.0f*focal / (r-l); @@ -525,11 +533,11 @@ void FilterOutputOpticalFlowPlugin::setupShadowAndColorTextures( GPU::Texture2D glPushAttrib( GL_TEXTURE_BIT ); + // Create and initialize the OpenGL texture object used to store the shadow map. if( !shadowMap.IsInstantiated() || shadowMap.Width() !=vp.X() || shadowMap.Height()!=vp.Y() ) { - // Create and initialize the OpenGL texture object used to store the shadow map. shadowMap.Create( GL_DEPTH_COMPONENT, vp.X(), vp.Y(), @@ -544,7 +552,7 @@ void FilterOutputOpticalFlowPlugin::setupShadowAndColorTextures( GPU::Texture2D } - // Loads the raster into the GPU as a texture image. + // Loads the raster to the GPU as a texture image. rr->bind(); QImage &img = rr->plane->image; GLubyte *rasterData = new GLubyte [ 3*img.width()*img.height() ]; @@ -569,9 +577,6 @@ void FilterOutputOpticalFlowPlugin::setupShadowAndColorTextures( GPU::Texture2D } -#if 0 -int aaaaa = 0; -#endif void FilterOutputOpticalFlowPlugin::paintShadowTexture( GPU::Texture2D &shadowMap, vcg::Matrix44f &proj, vcg::Matrix44f &pose ) @@ -603,20 +608,6 @@ void FilterOutputOpticalFlowPlugin::paintShadowTexture( GPU::Texture2D &shadowMa m_MeshVBO.Unbind(); fbuffer.Unbind(); -#if 0 -float *bf = new float [ fbuffer.Width()*fbuffer.Height() ]; -fbuffer.DumpTo( GL_DEPTH_ATTACHMENT, bf, GL_DEPTH_COMPONENT, GL_FLOAT ); -QImage img( fbuffer.Width(), fbuffer.Height(), QImage::Format_RGB888 ); -for( int y=fbuffer.Height()-1, n=0; y>=0; --y ) - for( int x=0; x> &validPairs ) { @@ -642,15 +633,14 @@ void FilterOutputOpticalFlowPlugin::retroProjection( RasterFaceMap &rasters, setupShader( reprojShader ); validPairs.clear(); - for( RasterFaceMap::iterator rproj=rasters.begin(); rproj!=rasters.end(); ++rproj ) + for( RasterFaceMap::iterator rproj=rpatches.begin(); rproj!=rpatches.end(); ++rproj ) { vcg::Matrix44f proj, pose, shadowProj; shadowTextureMatrices( rproj.key(), proj, pose, shadowProj ); setupShadowAndColorTextures( shadowMap, colorMap, rproj.key() ); paintShadowTexture( shadowMap, proj, pose ); -#if 1 - for( RasterFaceMap::iterator rref=rasters.begin(); rref!=rasters.end(); ++rref ) + for( RasterFaceMap::iterator rref=rpatches.begin(); rref!=rpatches.end(); ++rref ) if( rref != rproj ) { const vcg::Point2i vp = rref.key()->shot.Intrinsics.ViewportPx; @@ -726,7 +716,6 @@ void FilterOutputOpticalFlowPlugin::retroProjection( RasterFaceMap &rasters, delete [] buffer; } -#endif } m_MeshVBO.Release(); diff --git a/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.h b/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.h index c754c907c..6a5ba7992 100644 --- a/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.h +++ b/src/plugins_experimental/filter_ouput_optical_flow/filter_output_optical_flow.h @@ -56,8 +56,8 @@ class FilterOutputOpticalFlowPlugin : public QObject, public MeshFilterInterface NeighbSet &neighb ) const; void getNeighbors( CFaceO *f, NeighbSet &neighb ) const; - void expands( FaceVec &faces, - int n ); + void expands( Patch &patch, + int nbGrows ); void setupMeshVBO(); void setupShader( GPU::Shader &shader ); @@ -71,7 +71,7 @@ class FilterOutputOpticalFlowPlugin : public QObject, public MeshFilterInterface void paintShadowTexture( GPU::Texture2D &shadowMap, vcg::Matrix44f &proj, vcg::Matrix44f &pose ); - void retroProjection( RasterFaceMap &faces, + void retroProjection( RasterFaceMap &rpatches, float coverageThreshold, QMap> &validPairs ); bool loadRasterList( QString &filename, @@ -79,7 +79,7 @@ class FilterOutputOpticalFlowPlugin : public QObject, public MeshFilterInterface void saveXMLProject( const QString &filename, MeshModel *mm, - RasterFaceMap &rasters, + RasterFaceMap &rpatches, QMap> &validPairs );