lucaspalomodevelop/meshlab
1
0
Fork 0
mirror of https://github.com/lucaspalomodevelop/meshlab.git synced 2026-03-15 17:14:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

ongoing adding of kyotocabinet

Browse Source
This commit is contained in:
Fabio Ganovelli ganovelli 2011-01-21 15:59:55 +00:00
parent b91449941c
commit 2ff4b01ee9
10 changed files with 401 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/meshlabplugins/edit_ocme/src/ocme/cell.cpp
View File

@ -1,7 +1,11 @@
#include "../utils/memory_debug.h"
// TODO include must be taken out of the way.....
#ifdef SIMPLE_DB
#include "../ooc_vector/io/ooc_chains.hpp"
#else
#include "../ooc_vector/io/ooc_chains_kcdb.hpp"
#endif
//...............................................
#include "cell.h"
@ -10,7 +14,7 @@
Cell::~Cell(){
if(impostor)
delete impostor;
#ifndef _TEMP_BUILDER_MODE_
ClearEditCommitAuxData();
ClearRenderAuxData();

78
src/meshlabplugins/edit_ocme/src/ocme/main.cpp
View File

@ -16,8 +16,12 @@
#include "import_ocm_ply.h"
#include "../ooc_vector/io/ooc_chains.hpp"
#ifdef SIMPLE_DB
#include "../ooc_vector/io/ooc_chains.hpp"
#else
#include "../ooc_vector/io/ooc_chains_kcdb.hpp"
#endif
#include <wrap/io_trimesh/import_ply.h>
#include <wrap/io_trimesh/import_dae.h>
@ -38,6 +42,9 @@
#include <vector>
#include <iostream>
#include <windows.h>
#include <kcpolydb.h>
int SearchDirectory(std::vector<std::wstring> &refvecFiles,
std::wstring &refcstrRootDirectory,
std::wstring &refcstrExtension,
@ -277,67 +284,6 @@ bool Interrupt(){
}
//
//void AddFromDisk(OCME * ocme, std::vector<std::string> files){
//
// for(unsigned int i = 0; i < files.size(); ++i){
//
// std::string name = files[i];
// unsigned int wh = name.find(std::string(".aln"));
//
// if(wh == name.length()-4)
// // it is an aln file
// {
// std::vector<std::pair<std::string,vcg::Matrix44f> > aln;
// LoadAln(name.c_str(),aln);
//
// for(unsigned int idm = 0; idm< aln.size() ;++idm){
// vcgMesh m;
// vcg::tri::io::ImporterPLY<vcgMesh>::Open(m,aln[idm].first.c_str());
// vcg::tri::UpdatePosition<vcgMesh>::Matrix(m,aln[idm].second);
// if(!m.face.empty()){
// ocme->AddMesh(m);
// }
// }
// }
// else
// // It is a ply file
// {
// vcgMesh m;
// struct stat buf;
// stat(name.c_str(),&buf);
// if(buf.st_size < 200 * (1<<20)){// if the file is less that 200MB load the mesh in memory and then add it
//
// int mask = 0;
//
// TIM::Begin(0);
//
// vcg::tri::io::ImporterPLY<vcgMesh>::LoadMask(name.c_str(),mask);
//
// AttributeMapper am;
// if(mask & vcg::tri::io::Mask::IOM_VERTCOLOR){
// m.vert.EnableColor();
// am.vert_attrs.push_back("Color4b");
// }
//
// vcg::tri::io::ImporterPLY<vcgMesh>::Open(m,name.c_str(),cb);
// if(!m.face.empty())
// ocme->AddMesh(m,am);
// }
// else
// {
// vcg::Matrix44f tra_ma;tra_ma.SetIdentity();
// // if the file is more that 50 MB build directly from file
// vcg::tri::io::ImporterOCMPLY<vcgMesh>::Open(m,meshona,name.c_str(),tra_ma,cb);
//
// }
//
// }
//
// }
// ocme->RemoveEmptyCells();
//}
//
void PrintStats(){
@ -393,6 +339,8 @@ void PrintStats(){
lgn->Push();
}
int
main (int argc,char **argv )
{
@ -429,6 +377,8 @@ main (int argc,char **argv )
{
int c;
int digit_optind = 0;
unsigned int meshadded = 0,
@ -578,9 +528,9 @@ main (int argc,char **argv )
if(overwrite_database)
meshona->Create((std::string(ocmename)/*+std::string(".socm")*/).c_str(),berkeley_page_size);
meshona->Create((std::string(ocmename) +std::string(".kch") ).c_str(),berkeley_page_size);
else
meshona->Open((std::string(ocmename)+std::string(".socm")).c_str());
meshona->Open((std::string(ocmename)+std::string(".kch")).c_str());
if(all_plys)

4
src/meshlabplugins/edit_ocme/src/ocme/ocme.cpp
View File

@ -3,7 +3,11 @@
#include "../utils/memory_debug.h"
// TODO include must be taken out of the way.....
#ifdef SIMPLE_DB
#include "../ooc_vector/io/ooc_chains.hpp"
#else
#include "../ooc_vector/io/ooc_chains_kcdb.hpp"
#endif
//...............................................

4
src/meshlabplugins/edit_ocme/src/ocme/ocme_impostor.cpp
View File

@ -3,7 +3,11 @@
#include "ocme_extract.h"
/* UGLY: this dependency is because of the templates..it should be removed*/
#ifdef SIMPLE_DB
#include "../ooc_vector/io/ooc_chains.hpp"
#else
#include "../ooc_vector/io/ooc_chains_kcdb.hpp"
#endif
#include <vcg/space/normal_extrapolation.h>

4
src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains.hpp
View File

@ -64,7 +64,9 @@ FetchChunk( typename Chain<TYPE>::Chunk & ck ){
res = ((SimpleDb*)extMemHnd)->Get(ck.pos, local_buffer,ck.SizeOfDisk());
else{
res = ((SimpleDb*)extMemHnd)->Get(key, local_buffer, ck.SizeOfDisk());
#ifdef SIMPLE_DB
ck.pos = res;
#endif
}
if(res.Void() )
printf("record not found");
@ -88,7 +90,7 @@ LoadAll(){
{
RAssert(MemDbg::CheckHeap(1));
unsigned char * local_buffer = NULL;
char * local_buffer = NULL;
const unsigned int & chunk_order = &ck-&(*chunks.begin());
std::string key = this->GetKey(chunk_order);

103
src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chains_kcdb.hpp Normal file
View File

@ -0,0 +1,103 @@
#ifndef __CHUNK_MEM_DISK_
#define __CHUNK_MEM_DISK_
#include "../utils/timing.h"
#include "../../ooc_vector/ooc_chains.h"
#include <limits.h>
#include <kcpolydb.h>
template <class TYPE> int Chain<TYPE>::
SaveChunk( typename Chain<TYPE>::Chunk & ck ){
TIM::Begin(13);
char * local_buffer = 0;
if(ck.savedOnce && this->saveOnce) return 0 ;
RAssert(MemDbg::CheckHeap(0));
const unsigned int & chunk_order = &ck-&(*chunks.begin());
std::string key = this->GetKey(chunk_order);
unsigned int siz = ck.Write(local_buffer);
((kyotocabinet::PolyDB*)extMemHnd)->set(key.c_str(),key.length(), local_buffer, siz);
ck.Written(local_buffer);
if(ck.size == this->params.chunkSize )
ck.savedOnce = true;
else
STAT::Inc(N_SAVED_PART_CH);
TIM::End(13);
STAT::Inc(N_SAVEDCH);
return ck.SizeOfMem();
}
template <class TYPE> void Chain<TYPE>::
RemoveChunk( typename Chain<TYPE>::Chunk & ck ){
RAssert(MemDbg::CheckHeap(0));
const unsigned int & chunk_order = &ck-&(*chunks.begin());
std::string name = this->GetKey(chunk_order);
((kyotocabinet::PolyDB*)extMemHnd)->remove(name);
}
template <class TYPE> void Chain<TYPE>::
FetchChunk( typename Chain<TYPE>::Chunk & ck ){
RAssert(MemDbg::CheckHeap(1));
TIM::Begin(12);
const unsigned int & chunk_order = &ck-&(*chunks.begin());
std::string key = this->GetKey(chunk_order);
char * local_buffer;
ck.AllocMem(local_buffer);
size_t size_buffer;
local_buffer = ((kyotocabinet::PolyDB*)extMemHnd)->get(key.c_str(),key.length(), &size_buffer);
RAssert(ck.SizeOfDisk() == size_buffer);
if(local_buffer )
printf("record not found");
ck.Read(local_buffer);
TIM::End(12);
STAT::Inc(N_LOADEDCH);
}
template <class TYPE> void Chain<TYPE>::
LoadAll(){
incore.resize(this->Size());
for(unsigned int ci = 0; ci < chunks.size(); ++ci)
{
typename Chain<TYPE>::Chunk & ck = chunks[ci];
char * ptr = (char*) &this->incore[ci*this->params.chunkSize];
if(ck.buffer){
memcpy(ptr,ck.buffer,sizeof(TYPE)*ck.size);
}else
{
RAssert(MemDbg::CheckHeap(1));
char * local_buffer = NULL;
const unsigned int & chunk_order = &ck-&(*chunks.begin());
std::string key = this->GetKey(chunk_order);
ck.AllocMem(local_buffer,ptr);
size_t size_buffer;
local_buffer =((kyotocabinet::PolyDB*)extMemHnd)->get(key.c_str(),key.length(),& size_buffer);
RAssert(local_buffer);
ck.Read(local_buffer,ptr);
}
}
}
#endif

104
src/meshlabplugins/edit_ocme/src/ooc_vector/io/ooc_chunks_kcdb.cpp Normal file
View File

@ -0,0 +1,104 @@
#include "../ooc_chains.h"
#include "../utils/timing.h"
extern Logging * lgn;
#include <kcpolydb.h>
using namespace std;
void OOCEnv::
Create( const char * name ){
is_file_owned = true;
std::string iDbName(name); // name of the database
extMemHnd = new kyotocabinet::PolyDB();
if(!((kyotocabinet::PolyDB*)extMemHnd)->open(iDbName, kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE))
{
printf("Failed to create databse: %s ",((kyotocabinet::PolyDB*)extMemHnd)->error().name());
exit(0);
}
}
void OOCEnv::
Create( void * handle ){
is_file_owned = false;
extMemHnd = (kyotocabinet::PolyDB*) handle;
}
void OOCEnv::
Open( const char * name ){
is_file_owned = true;
extMemHnd = new kyotocabinet::PolyDB();
((kyotocabinet::PolyDB*)extMemHnd)->open(name,kyotocabinet::PolyDB::OWRITER|kyotocabinet::PolyDB::OREADER);
LoadAT();
}
void OOCEnv::
Open( void * handle ){
is_file_owned = false;
extMemHnd = (kyotocabinet::PolyDB*) handle;
LoadAT();
}
void OOCEnv::
Close( bool andsave){
if(andsave){
SaveData();
// ((SimpleDb*)extMemHnd)->DisableSafeWriting();
SaveAT();
}
lgn->Append("deleting kcdb ");
delete (kyotocabinet::PolyDB*)extMemHnd;
lgn->Append("done");
}
int OOCEnv::
SaveAT( ){
int siz = this->SizeOfMem(); // size of the Allocation Table
char * buf = new char[siz]; // allocate memory
#ifdef _DEBUG
char * res =
#endif
this->Serialize(buf); // serialize
#ifdef _DEBUG
RAssert(res-buf == siz);
#endif
((kyotocabinet::PolyDB*)extMemHnd)->set("CHAINMEM_ALLOCATION_TABLE",25,buf,siz);
delete [] buf;
sprintf(lgn->Buf(),"Allocation Table Size: %d",siz);
lgn->Push();
sprintf(lgn->Buf() ,"mem size:%d",siz);
lgn->Push();
return siz;
}
void OOCEnv::
LoadAT( ){
std::string key("CHAINMEM_ALLOCATION_TABLE");
unsigned int size;
size_t size_buf;
char * buf = ((kyotocabinet::PolyDB*)extMemHnd)->get("CHAINMEM_ALLOCATION_TABLE",25, &size_buf);
RAssert(buf);
this->DeSerialize((char*)buf);
size = this->SizeOfMem();
delete [] (char*) buf;
}

6
src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.cpp
View File

@ -1,6 +1,10 @@
#include "ooc_chains.h"
#include "./io/ooc_chains.hpp"
#ifdef SIMPLE_DB
#include "./io/ooc_chains.hpp"
#else
#include "./io/ooc_chains_kcdb.hpp"
#endif
/* -------------------------------------------------------------------------- */
/* ------------------------- OOCEnv implementation ----------------------------- */

27
src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains.h
View File

@ -19,8 +19,9 @@
#include "../utils/memory_debug.h"
#include "../utils/release_assert.h"
#include <limits>
#ifdef SIMPLE_DB
#include "../ooc_vector/simpledb.h"
#endif
/* **********************************************************************
the classes Chain and OOCEnv implement very large vector like containers (Chain)
that do not fit in memory.
@ -129,7 +130,7 @@ struct Chain: public ChainBase{
struct Chunk{
Chunk():buffer(0),fetch_time(0),savedOnce(false){
#ifdef NO_BERKELEY
#ifdef SIMPLE_DB
pos.SetVoid();
#endif
}
@ -142,7 +143,9 @@ struct Chain: public ChainBase{
int fetch_time; // the last time it was loaded
bool savedOnce; // if the chunk has been saved at least once
unsigned int size_of_disk; // size on disk
#ifdef SIMPLE_DB
SimpleDb::Index pos;
#endif
/* query */
bool IsLoaded(){return buffer!=NULL;}
@ -153,13 +156,13 @@ struct Chain: public ChainBase{
/* read/write */
unsigned int SizeOfDisk();
unsigned int SizeOfMem();
void Written(unsigned char * & buffer);
unsigned int Write(unsigned char * & buffer);
void Read(unsigned char * buffer, unsigned char * here = NULL);
void Written( char * & buffer);
unsigned int Write( char * & buffer);
void Read( char * buffer, char * here = NULL);
static float CompressionRatio();
/* allocation */
void AllocMem(unsigned char *& buf,unsigned char * here = NULL);
void AllocMem( char *& buf, char * here = NULL);
void DeAllocMem();
};
public:
@ -815,18 +818,18 @@ SizeOfDisk(){ return sizeof(TYPE)*this->size;}
template <class TYPE> void Chain<TYPE>::Chunk::
Written(unsigned char * & buffer){buffer=buffer;}
Written( char * & buffer){buffer=buffer;}
template <class TYPE> unsigned int Chain<TYPE>::Chunk::
Write(unsigned char * & buffer ){
buffer = (unsigned char*) this->buffer;
Write(char * & buffer ){
buffer = ( char*) this->buffer;
this->size_of_disk = this->SizeOfMem();
return this->size_of_disk;
}
template <class TYPE> void Chain<TYPE>::Chunk::
Read(unsigned char *buf,unsigned char *here ){
Read( char *buf, char *here ){
if(!here)
this->buffer = (TYPE*)buf;
}
@ -835,10 +838,10 @@ template <class TYPE> float Chain<TYPE>::Chunk::
CompressionRatio(){return 1.0;};
template <class TYPE> void Chain<TYPE>::Chunk::
AllocMem(unsigned char * & buf,unsigned char * here ){
AllocMem( char * & buf, char * here ){
if(!here){
this->buffer = new TYPE[this->capacity]; // allocate the internal memory of the chunk
buf = (unsigned char*) this->buffer; // return
buf = ( char*) this->buffer; // return
}else
buf = here; // simply copy the passed pointer
};

144
src/meshlabplugins/edit_ocme/src/ooc_vector/ooc_chains_kcdb.cpp Normal file
View File

@ -0,0 +1,144 @@
#include "ooc_chains.h"
#include "./io/ooc_chains_kcdb.hpp"
/* -------------------------------------------------------------------------- */
/* ------------------------- OOCEnv implementation ----------------------------- */
/* -------------------------------------------------------------------------- */
void
OOCEnv::TrashEverything(){
cache_policy->FlushAllChunks(false);
Close(false);
}
void
OOCEnv ::InitDefaultNameTypeBinders(){
this->AddNameTypeBound<float>("float");
this->AddNameTypeBound<int>("int");
this->AddNameTypeBound<char>("char");
this->AddNameTypeBound<short>("short");
this->AddNameTypeBound<double>("double");
this->AddNameTypeBound<unsigned int>("unsigned int");
this->AddNameTypeBound<unsigned short>("unsigned short");
this->AddNameTypeBound<unsigned char>("unsigned char");
}
ChainBase *
OOCEnv ::CreateFromString(const std::string & key, const std::string & type ){
ChainBase * newchain;
BoundsIterator i;
for(i = binders.begin(); i != binders.end(); ++i)
if((*i).second->Name() == type) {newchain = (*i).second->NewChain();break;}
RAssert(i != binders.end()); // none of the admitted types
newchain->Key () = key;
newchain->Type() = type;
newchain->CachePol() = this->cache_policy;
newchain->ExtMemHnd() = this->extMemHnd;
newchain->SetChunkSize(params.blockSizeBytes);
return newchain;
}
ChainBase *
OOCEnv ::
GetChain( std::string name ){
ChainIterator ci;
ci = chains.find(name);
if( ci != chains.end())
return (*ci).second;
return NULL; // it does not exists
}
void
OOCEnv ::
RemoveChain( const std::string & name ){
ChainBase * c = GetChain(name);
RAssert( c != NULL);
c->RemoveAllChunks(); // remove all the chunks
chains.erase(name); // remove the chain from the map of all the chains
c->PointedBy(-1); // delete the datastructure
// SaveAT(); // save the updated allocation table
}
void OOCEnv ::
SaveData(){
cache_policy->FlushAllChunks();
//for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci)
//(*ci).second->SaveData();
}
/* SERIALIZATION */
/* OOCEnv */
int OOCEnv ::
SizeOfMem(){
BoundsIterator bi;
int siz = 0;
siz += sizeof(Params);
siz += sizeof(unsigned int); // number of binders
for(bi = binders.begin(); bi != binders.end(); ++bi) siz += (*bi).second->SizeOfMem();
siz += sizeof(int); // number of chains
for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci)
siz +=(*ci).second->SizeOfMem();
return siz;
}
char * OOCEnv ::
Serialize(char * buffer){
BoundsIterator bi;
char * ptr = buffer;
memcpy(ptr,&params,sizeof(Params) );ptr+=sizeof(Params) ;
*(unsigned int*)ptr = binders.size(); ptr+=sizeof(unsigned int);
for(bi = binders.begin(); bi != binders.end(); ++bi) ptr = (*bi).second->Serialize(ptr);
unsigned int n_chain = (int)chains.size();
memcpy(ptr,&n_chain,sizeof(int));ptr+=sizeof(int);
for( ChainIterator ci = chains.begin(); ci != chains.end(); ++ci)
ptr = (*ci).second->Serialize(ptr);
return ptr;
}
char * OOCEnv ::
DeSerialize(char * buffer){
unsigned int i;
char * ptr = buffer;
memcpy(&params,ptr,sizeof(Params) );ptr+=sizeof(Params) ;
unsigned int n_binders = *(unsigned int*)ptr; ptr+=sizeof(unsigned int);
for(i = 0; i < n_binders; ++i) {
NameUnknownTypeBind tb;
ptr = tb.DeSerialize(ptr);
AddNameUnknownTypeBound(tb);
}
int n_chain;
memcpy(&n_chain,ptr,sizeof(int));ptr+=sizeof(int);
std::string key,type;
std::pair<ChainIterator,bool> ins;
unsigned int mem;
for( int ic = 0; ic <n_chain;++ic){
MemDbg::SetPoint(0);
char * ptrB = ptr; // the pointer is saved in ptrB and then passed to DeSerialize
ptr = ::DeSerialize(ptr,key); // This is done to read "name" and "type" an to be able to create a chain
ptr = ::DeSerialize(ptr,type);
ins = chains.insert(std::pair<std::string,ChainBase*>(key,CreateFromString(key,type))); // here the chain is allocated
ptr = (*(ins.first)).second->DeSerialize(ptrB); // Here DeSerialize with the saved poiner is called
mem = MemDbg::MemFromPoint(0);
}
return ptr;
}