diff --git a/src/meshlabplugins/edit_ocme/src/ocme/cell.cpp b/src/meshlabplugins/edit_ocme/src/ocme/cell.cpp index fc0c3730a..db6437282 100755 --- a/src/meshlabplugins/edit_ocme/src/ocme/cell.cpp +++ b/src/meshlabplugins/edit_ocme/src/ocme/cell.cpp @@ -1,7 +1,11 @@ #include "../utils/memory_debug.h" // TODO include must be taken out of the way..... +#ifdef SIMPLE_DB #include "../ooc_vector/io/ooc_chains.hpp" +#else +#include "../ooc_vector/io/ooc_chains_kcdb.hpp" +#endif //............................................... #include "cell.h" @@ -10,7 +14,7 @@ Cell::~Cell(){ if(impostor) delete impostor; - + #ifndef _TEMP_BUILDER_MODE_ ClearEditCommitAuxData(); ClearRenderAuxData(); diff --git a/src/meshlabplugins/edit_ocme/src/ocme/main.cpp b/src/meshlabplugins/edit_ocme/src/ocme/main.cpp index ad4748381..0a8325dbf 100755 --- a/src/meshlabplugins/edit_ocme/src/ocme/main.cpp +++ b/src/meshlabplugins/edit_ocme/src/ocme/main.cpp @@ -16,8 +16,12 @@ #include "import_ocm_ply.h" -#include "../ooc_vector/io/ooc_chains.hpp" +#ifdef SIMPLE_DB +#include "../ooc_vector/io/ooc_chains.hpp" +#else +#include "../ooc_vector/io/ooc_chains_kcdb.hpp" +#endif #include #include @@ -38,6 +42,9 @@ #include #include #include + +#include + int SearchDirectory(std::vector &refvecFiles, std::wstring &refcstrRootDirectory, std::wstring &refcstrExtension, @@ -277,67 +284,6 @@ bool Interrupt(){ } -// -//void AddFromDisk(OCME * ocme, std::vector files){ -// -// for(unsigned int i = 0; i < files.size(); ++i){ -// -// std::string name = files[i]; -// unsigned int wh = name.find(std::string(".aln")); -// -// if(wh == name.length()-4) -// // it is an aln file -// { -// std::vector > aln; -// LoadAln(name.c_str(),aln); -// -// for(unsigned int idm = 0; idm< aln.size() ;++idm){ -// vcgMesh m; -// vcg::tri::io::ImporterPLY::Open(m,aln[idm].first.c_str()); -// vcg::tri::UpdatePosition::Matrix(m,aln[idm].second); -// if(!m.face.empty()){ -// ocme->AddMesh(m); -// } -// } -// } -// else -// // It is a ply file -// { -// vcgMesh m; -// struct stat buf; -// stat(name.c_str(),&buf); -// if(buf.st_size < 200 * (1<<20)){// if the file is less that 200MB load the mesh in memory and then add it -// -// int mask = 0; -// -// TIM::Begin(0); -// -// vcg::tri::io::ImporterPLY::LoadMask(name.c_str(),mask); -// -// AttributeMapper am; -// if(mask & vcg::tri::io::Mask::IOM_VERTCOLOR){ -// m.vert.EnableColor(); -// am.vert_attrs.push_back("Color4b"); -// } -// -// vcg::tri::io::ImporterPLY::Open(m,name.c_str(),cb); -// if(!m.face.empty()) -// ocme->AddMesh(m,am); -// } -// else -// { -// vcg::Matrix44f tra_ma;tra_ma.SetIdentity(); -// // if the file is more that 50 MB build directly from file -// vcg::tri::io::ImporterOCMPLY::Open(m,meshona,name.c_str(),tra_ma,cb); -// -// } -// -// } -// -// } -// ocme->RemoveEmptyCells(); -//} -// void PrintStats(){ @@ -393,6 +339,8 @@ void PrintStats(){ lgn->Push(); } + + int main (int argc,char **argv ) { @@ -429,6 +377,8 @@ main (int argc,char **argv ) { + + int c; int digit_optind = 0; unsigned int meshadded = 0, @@ -578,9 +528,9 @@ main (int argc,char **argv ) if(overwrite_database) - meshona->Create((std::string(ocmename)/*+std::string(".socm")*/).c_str(),berkeley_page_size); + meshona->Create((std::string(ocmename) +std::string(".kch") ).c_str(),berkeley_page_size); else - meshona->Open((std::string(ocmename)+std::string(".socm")).c_str()); + meshona->Open((std::string(ocmename)+std::string(".kch")).c_str()); if(all_plys) diff --git a/src/meshlabplugins/edit_ocme/src/ocme/ocme.cpp b/src/meshlabplugins/edit_ocme/src/ocme/ocme.cpp index e21be1847..a55050972 100755 --- a/src/meshlabplugins/edit_ocme/src/ocme/ocme.cpp +++ b/src/meshlabplugins/edit_ocme/src/ocme/ocme.cpp @@ -3,7 +3,11 @@ #include "../utils/memory_debug.h" // TODO include must be taken out of the way..... +#ifdef SIMPLE_DB #include "../ooc_vector/io/ooc_chains.hpp" +#else +#include "../ooc_vector/io/ooc_chains_kcdb.hpp" +#endif //............................................... diff --git a/src/meshlabplugins/edit_ocme/src/ocme/ocme_impostor.cpp b/src/meshlabplugins/edit_ocme/src/ocme/ocme_impostor.cpp index da1b8e1a4..4bc7822b8 100755 --- a/src/meshlabplugins/edit_ocme/src/ocme/ocme_impostor.cpp +++ b/src/meshlabplugins/edit_ocme/src/ocme/ocme_impostor.cpp @@ -3,7 +3,11 @@ #include "ocme_extract.h" /* UGLY: this dependency is because of the templates..it should be removed*/ +#ifdef SIMPLE_DB #include "../ooc_vector/io/ooc_chains.hpp" +#else +#include "../ooc_vector/io/ooc_chains_kcdb.hpp" +#endif #include diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains.hpp b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains.hpp index fbe835c50..455296181 100755 --- a/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains.hpp +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains.hpp @@ -64,7 +64,9 @@ FetchChunk( typename Chain::Chunk & ck ){ res = ((SimpleDb*)extMemHnd)->Get(ck.pos, local_buffer,ck.SizeOfDisk()); else{ res = ((SimpleDb*)extMemHnd)->Get(key, local_buffer, ck.SizeOfDisk()); +#ifdef SIMPLE_DB ck.pos = res; +#endif } if(res.Void() ) printf("record not found"); @@ -88,7 +90,7 @@ LoadAll(){ { RAssert(MemDbg::CheckHeap(1)); - unsigned char * local_buffer = NULL; + char * local_buffer = NULL; const unsigned int & chunk_order = &ck-&(*chunks.begin()); std::string key = this->GetKey(chunk_order); diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains_kcdb.hpp b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains_kcdb.hpp new file mode 100644 index 000000000..77e947cd3 --- /dev/null +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains_kcdb.hpp @@ -0,0 +1,103 @@ +#ifndef __CHUNK_MEM_DISK_ +#define __CHUNK_MEM_DISK_ +#include "../utils/timing.h" +#include "../../ooc_vector/ooc_chains.h" +#include + +#include + +template int Chain:: +SaveChunk( typename Chain::Chunk & ck ){ + TIM::Begin(13); + char * local_buffer = 0; + + if(ck.savedOnce && this->saveOnce) return 0 ; + + RAssert(MemDbg::CheckHeap(0)); + + const unsigned int & chunk_order = &ck-&(*chunks.begin()); + std::string key = this->GetKey(chunk_order); + + unsigned int siz = ck.Write(local_buffer); + + + ((kyotocabinet::PolyDB*)extMemHnd)->set(key.c_str(),key.length(), local_buffer, siz); + + ck.Written(local_buffer); + + if(ck.size == this->params.chunkSize ) + ck.savedOnce = true; + else + STAT::Inc(N_SAVED_PART_CH); + + TIM::End(13); + STAT::Inc(N_SAVEDCH); + return ck.SizeOfMem(); +} + +template void Chain:: +RemoveChunk( typename Chain::Chunk & ck ){ + RAssert(MemDbg::CheckHeap(0)); + const unsigned int & chunk_order = &ck-&(*chunks.begin()); + std::string name = this->GetKey(chunk_order); + + ((kyotocabinet::PolyDB*)extMemHnd)->remove(name); +} + + +template void Chain:: +FetchChunk( typename Chain::Chunk & ck ){ + RAssert(MemDbg::CheckHeap(1)); + TIM::Begin(12); + + const unsigned int & chunk_order = &ck-&(*chunks.begin()); + std::string key = this->GetKey(chunk_order); + + char * local_buffer; + + ck.AllocMem(local_buffer); + + size_t size_buffer; + local_buffer = ((kyotocabinet::PolyDB*)extMemHnd)->get(key.c_str(),key.length(), &size_buffer); + + RAssert(ck.SizeOfDisk() == size_buffer); + + if(local_buffer ) + printf("record not found"); + + ck.Read(local_buffer); + + TIM::End(12); + STAT::Inc(N_LOADEDCH); +} + +template void Chain:: +LoadAll(){ + incore.resize(this->Size()); + for(unsigned int ci = 0; ci < chunks.size(); ++ci) + { + typename Chain::Chunk & ck = chunks[ci]; + char * ptr = (char*) &this->incore[ci*this->params.chunkSize]; + if(ck.buffer){ + memcpy(ptr,ck.buffer,sizeof(TYPE)*ck.size); + }else + { + RAssert(MemDbg::CheckHeap(1)); + + char * local_buffer = NULL; + const unsigned int & chunk_order = &ck-&(*chunks.begin()); + std::string key = this->GetKey(chunk_order); + + ck.AllocMem(local_buffer,ptr); + + size_t size_buffer; + local_buffer =((kyotocabinet::PolyDB*)extMemHnd)->get(key.c_str(),key.length(),& size_buffer); + RAssert(local_buffer); + + ck.Read(local_buffer,ptr); + } + } +} + + +#endif diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chunks_kcdb.cpp b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chunks_kcdb.cpp new file mode 100644 index 000000000..9b7873d2b --- /dev/null +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chunks_kcdb.cpp @@ -0,0 +1,104 @@ +#include "../ooc_chains.h" +#include "../utils/timing.h" + + +extern Logging * lgn; + + +#include +using namespace std; + + + +void OOCEnv:: +Create( const char * name ){ + is_file_owned = true; + + std::string iDbName(name); // name of the database + extMemHnd = new kyotocabinet::PolyDB(); + if(!((kyotocabinet::PolyDB*)extMemHnd)->open(iDbName, kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE)) + { + printf("Failed to create databse: %s ",((kyotocabinet::PolyDB*)extMemHnd)->error().name()); + exit(0); + } + +} + +void OOCEnv:: +Create( void * handle ){ + is_file_owned = false; + extMemHnd = (kyotocabinet::PolyDB*) handle; +} + +void OOCEnv:: +Open( const char * name ){ + is_file_owned = true; + extMemHnd = new kyotocabinet::PolyDB(); + ((kyotocabinet::PolyDB*)extMemHnd)->open(name,kyotocabinet::PolyDB::OWRITER|kyotocabinet::PolyDB::OREADER); + LoadAT(); +} + +void OOCEnv:: +Open( void * handle ){ + is_file_owned = false; + extMemHnd = (kyotocabinet::PolyDB*) handle; + LoadAT(); +} + +void OOCEnv:: +Close( bool andsave){ + if(andsave){ + SaveData(); + // ((SimpleDb*)extMemHnd)->DisableSafeWriting(); + SaveAT(); + } + lgn->Append("deleting kcdb "); + delete (kyotocabinet::PolyDB*)extMemHnd; + lgn->Append("done"); + +} + + +int OOCEnv:: +SaveAT( ){ + + int siz = this->SizeOfMem(); // size of the Allocation Table + char * buf = new char[siz]; // allocate memory +#ifdef _DEBUG + char * res = +#endif + this->Serialize(buf); // serialize +#ifdef _DEBUG + RAssert(res-buf == siz); +#endif + + + + ((kyotocabinet::PolyDB*)extMemHnd)->set("CHAINMEM_ALLOCATION_TABLE",25,buf,siz); + delete [] buf; + + sprintf(lgn->Buf(),"Allocation Table Size: %d",siz); + lgn->Push(); + + sprintf(lgn->Buf() ,"mem size:%d",siz); + lgn->Push(); + + return siz; +} + +void OOCEnv:: +LoadAT( ){ + std::string key("CHAINMEM_ALLOCATION_TABLE"); + + unsigned int size; + size_t size_buf; + char * buf = ((kyotocabinet::PolyDB*)extMemHnd)->get("CHAINMEM_ALLOCATION_TABLE",25, &size_buf); + RAssert(buf); + + + this->DeSerialize((char*)buf); + size = this->SizeOfMem(); + + delete [] (char*) buf; +} + diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.cpp b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.cpp index c1e178bac..233d1914b 100755 --- a/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.cpp +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.cpp @@ -1,6 +1,10 @@ #include "ooc_chains.h" -#include "./io/ooc_chains.hpp" +#ifdef SIMPLE_DB +#include "./io/ooc_chains.hpp" +#else +#include "./io/ooc_chains_kcdb.hpp" +#endif /* -------------------------------------------------------------------------- */ /* ------------------------- OOCEnv implementation ----------------------------- */ diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.h b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.h index 51c848eae..53be275d9 100755 --- a/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.h +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.h @@ -19,8 +19,9 @@ #include "../utils/memory_debug.h" #include "../utils/release_assert.h" #include +#ifdef SIMPLE_DB #include "../ooc_vector/simpledb.h" - +#endif /* ********************************************************************** the classes Chain and OOCEnv implement very large vector like containers (Chain) that do not fit in memory. @@ -129,7 +130,7 @@ struct Chain: public ChainBase{ struct Chunk{ Chunk():buffer(0),fetch_time(0),savedOnce(false){ -#ifdef NO_BERKELEY +#ifdef SIMPLE_DB pos.SetVoid(); #endif } @@ -142,7 +143,9 @@ struct Chain: public ChainBase{ int fetch_time; // the last time it was loaded bool savedOnce; // if the chunk has been saved at least once unsigned int size_of_disk; // size on disk +#ifdef SIMPLE_DB SimpleDb::Index pos; +#endif /* query */ bool IsLoaded(){return buffer!=NULL;} @@ -153,13 +156,13 @@ struct Chain: public ChainBase{ /* read/write */ unsigned int SizeOfDisk(); unsigned int SizeOfMem(); - void Written(unsigned char * & buffer); - unsigned int Write(unsigned char * & buffer); - void Read(unsigned char * buffer, unsigned char * here = NULL); + void Written( char * & buffer); + unsigned int Write( char * & buffer); + void Read( char * buffer, char * here = NULL); static float CompressionRatio(); /* allocation */ - void AllocMem(unsigned char *& buf,unsigned char * here = NULL); + void AllocMem( char *& buf, char * here = NULL); void DeAllocMem(); }; public: @@ -815,18 +818,18 @@ SizeOfDisk(){ return sizeof(TYPE)*this->size;} template void Chain::Chunk:: -Written(unsigned char * & buffer){buffer=buffer;} +Written( char * & buffer){buffer=buffer;} template unsigned int Chain::Chunk:: -Write(unsigned char * & buffer ){ - buffer = (unsigned char*) this->buffer; +Write(char * & buffer ){ + buffer = ( char*) this->buffer; this->size_of_disk = this->SizeOfMem(); return this->size_of_disk; } template void Chain::Chunk:: -Read(unsigned char *buf,unsigned char *here ){ +Read( char *buf, char *here ){ if(!here) this->buffer = (TYPE*)buf; } @@ -835,10 +838,10 @@ template float Chain::Chunk:: CompressionRatio(){return 1.0;}; template void Chain::Chunk:: -AllocMem(unsigned char * & buf,unsigned char * here ){ +AllocMem( char * & buf, char * here ){ if(!here){ this->buffer = new TYPE[this->capacity]; // allocate the internal memory of the chunk - buf = (unsigned char*) this->buffer; // return + buf = ( char*) this->buffer; // return }else buf = here; // simply copy the passed pointer }; diff --git a/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains_kcdb.cpp b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains_kcdb.cpp new file mode 100644 index 000000000..5d8240f2b --- /dev/null +++ b/src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains_kcdb.cpp @@ -0,0 +1,144 @@ +#include "ooc_chains.h" +#include "./io/ooc_chains_kcdb.hpp" + + +/* -------------------------------------------------------------------------- */ +/* ------------------------- OOCEnv implementation ----------------------------- */ +/* -------------------------------------------------------------------------- */ + +void +OOCEnv::TrashEverything(){ + cache_policy->FlushAllChunks(false); + Close(false); +} + +void +OOCEnv ::InitDefaultNameTypeBinders(){ + this->AddNameTypeBound("float"); + this->AddNameTypeBound("int"); + this->AddNameTypeBound("char"); + this->AddNameTypeBound("short"); + this->AddNameTypeBound("double"); + this->AddNameTypeBound("unsigned int"); + this->AddNameTypeBound("unsigned short"); + this->AddNameTypeBound("unsigned char"); +} + + +ChainBase * +OOCEnv ::CreateFromString(const std::string & key, const std::string & type ){ + ChainBase * newchain; + BoundsIterator i; + for(i = binders.begin(); i != binders.end(); ++i) + if((*i).second->Name() == type) {newchain = (*i).second->NewChain();break;} + + RAssert(i != binders.end()); // none of the admitted types + + newchain->Key () = key; + newchain->Type() = type; + newchain->CachePol() = this->cache_policy; + newchain->ExtMemHnd() = this->extMemHnd; + newchain->SetChunkSize(params.blockSizeBytes); + return newchain; + +} + + +ChainBase * +OOCEnv :: +GetChain( std::string name ){ + ChainIterator ci; + ci = chains.find(name); + if( ci != chains.end()) + return (*ci).second; + + return NULL; // it does not exists +} + +void +OOCEnv :: +RemoveChain( const std::string & name ){ + ChainBase * c = GetChain(name); + RAssert( c != NULL); + c->RemoveAllChunks(); // remove all the chunks + chains.erase(name); // remove the chain from the map of all the chains + c->PointedBy(-1); // delete the datastructure +// SaveAT(); // save the updated allocation table + } + + + +void OOCEnv :: +SaveData(){ + cache_policy->FlushAllChunks(); + //for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci) + //(*ci).second->SaveData(); +} + + +/* SERIALIZATION */ +/* OOCEnv */ +int OOCEnv :: +SizeOfMem(){ + BoundsIterator bi; + int siz = 0; + siz += sizeof(Params); + siz += sizeof(unsigned int); // number of binders + for(bi = binders.begin(); bi != binders.end(); ++bi) siz += (*bi).second->SizeOfMem(); + siz += sizeof(int); // number of chains + for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci) + siz +=(*ci).second->SizeOfMem(); + return siz; +} + +char * OOCEnv :: +Serialize(char * buffer){ + BoundsIterator bi; + char * ptr = buffer; + memcpy(ptr,¶ms,sizeof(Params) );ptr+=sizeof(Params) ; + + *(unsigned int*)ptr = binders.size(); ptr+=sizeof(unsigned int); + for(bi = binders.begin(); bi != binders.end(); ++bi) ptr = (*bi).second->Serialize(ptr); + + unsigned int n_chain = (int)chains.size(); + memcpy(ptr,&n_chain,sizeof(int));ptr+=sizeof(int); + + for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci) + ptr = (*ci).second->Serialize(ptr); + return ptr; +} + +char * OOCEnv :: +DeSerialize(char * buffer){ + unsigned int i; + + char * ptr = buffer; + memcpy(¶ms,ptr,sizeof(Params) );ptr+=sizeof(Params) ; + + unsigned int n_binders = *(unsigned int*)ptr; ptr+=sizeof(unsigned int); + for(i = 0; i < n_binders; ++i) { + NameUnknownTypeBind tb; + ptr = tb.DeSerialize(ptr); + AddNameUnknownTypeBound(tb); + } + + int n_chain; + memcpy(&n_chain,ptr,sizeof(int));ptr+=sizeof(int); + + std::string key,type; + std::pair ins; + + unsigned int mem; + for( int ic = 0; ic (key,CreateFromString(key,type))); // here the chain is allocated + ptr = (*(ins.first)).second->DeSerialize(ptrB); // Here DeSerialize with the saved poiner is called + mem = MemDbg::MemFromPoint(0); + } + + return ptr; +} +