Added compression and debugged.

apps/nexus/nexus.cpp

@@ -11,7 +11,7 @@ Nexus::~Nexus() {
   Close();
 }
 
-bool Nexus::Create(const string &file, Signature sig) {
+bool Nexus::Create(const string &file, Signature sig, unsigned int c_size) {
   index_file = fopen((file + ".nxs").c_str(), "wb+");
   if(!index_file) {
     cerr << "Could not create file: " << file << ".nxs\n";
@@ -25,7 +25,7 @@ bool Nexus::Create(const string &file, Signature sig) {
   
   index.clear();
 
-  chunk_size = 1024;
+  chunk_size = c_size;
 
   if(!patches.Create(file + ".nxp", signature, chunk_size)) {
     cerr << "Could not create file: " << file << ".nxp" << endl;
@@ -96,6 +96,9 @@ bool Nexus::Load(const string &file, bool readonly) {
 }
 
 void Nexus::Close() {  
+  patches.Close();
+  borders.Close();
+
   if(!index_file) return;
   rewind(index_file);
 
@@ -131,9 +134,6 @@ void Nexus::Close() {
 
   fclose(index_file);
   index_file = NULL;
-
-  patches.Close();
-  borders.Close();
 }
 
 Patch &Nexus::GetPatch(unsigned int patch, bool flush) {

apps/nexus/nexus.h

@@ -50,7 +50,8 @@ class Nexus {
   Nexus();
   virtual ~Nexus();
 
-  bool Create(const std::string &filename, Signature signature);
+  bool Create(const std::string &filename, Signature signature,
+	      unsigned int chunk_size = 1024);
   virtual bool Load(const std::string &filename, bool readonly = false);
   virtual void Close();
 
@@ -77,6 +78,7 @@ class Nexus {
   /* Nexus data */
   
   //BE CAREFUL: this 2 members get replicated into patchserver
+  //TODO fix this nasty thing it is dangerous as it is.
   Signature signature;
   unsigned int chunk_size;
   

apps/nexus/nexusmt.cpp

@@ -41,8 +41,8 @@ NexusMt::NexusMt(): vbo(VBO_AUTO), vbo_size(0),
 
 NexusMt::~NexusMt() {}
 
-bool NexusMt::Load(const string &filename) {
+bool NexusMt::Load(const string &filename, bool readonly) {
-  if(!Nexus::Load(filename)) return false;
+  if(!Nexus::Load(filename, readonly)) return false;
   LoadHistory();
 
   use_colors = false;

apps/nexus/nexusmt.h

@@ -83,7 +83,7 @@ class NexusMt: public Nexus {
   NexusMt();
   ~NexusMt();
   
-  bool Load(const std::string &filename);
+  bool Load(const std::string &filename, bool readonly = true);
   bool InitGL();
   
   void Render();

apps/nexus/nxsedit.cpp

@@ -30,6 +30,8 @@ int main(int argc, char *argv[]) {
   string output;
   string plysource;
   bool info = false;
+  unsigned int ram_size = 128000000;
+  unsigned int chunk_size = 0;
 
   unsigned int add = 0;
   bool add_strip = false;
@@ -53,7 +55,7 @@ int main(int argc, char *argv[]) {
   float qtexture = 0;
 
   int option;
-  while((option = getopt(argc, argv, "io:a:r:zxv:n:c:t:")) != EOF) {
+  while((option = getopt(argc, argv, "io:a:r:zxv:n:k:t:b:c:")) != EOF) {
     switch(option) {
     case 'i': info = true; break;
     case 'o': output = optarg; break;
@@ -135,7 +137,7 @@ int main(int argc, char *argv[]) {
 	return -1;
       }
       break;
-    case 'c': qcolor = atof(optarg); 
+    case 'k': qcolor = atof(optarg); 
       if(qcolor == 0) {
 	cerr << "Invalid value for quantization: " << optarg << endl;
 	return -1;
@@ -147,7 +149,18 @@ int main(int argc, char *argv[]) {
 	return -1;
       }
       break;
-
+    case 'b': ram_size = atoi(optarg); 
+      if(ram_size == 0) {
+	cerr << "Invalid ram_size: " << optarg << endl;
+	return -1;
+      }
+      break;
+    case 'c': chunk_size = atoi(optarg); 
+      if(chunk_size == 0) {
+	cerr << "Invalid chunk_size: " << optarg << endl;
+	return -1;
+      }
+      break;
     default: cerr << "Unknown option: " << (char)option << endl;
       return -1;
     }
@@ -177,11 +190,13 @@ int main(int argc, char *argv[]) {
 
 
   Nexus nexus;
-  if(!nexus.Load(input)) {
+  nexus.patches.SetRamBufferSize(ram_size);
+  if(!nexus.Load(input, true)) {
     cerr << "Could not open nexus file: " << input << ".mt\n";
     return -1;
   }
 
+
   //Sanity tests
   if(remove_strip && !(nexus.signature & NXS_STRIP)) {
     cerr << "Nexus file does not have strips\n";
@@ -226,7 +241,9 @@ int main(int argc, char *argv[]) {
 	 << "\n\tData    : " << (int)((nexus.signature&NXS_DATA32) !=0)
 	 << "\n\n\tVertices: " << nexus.totvert 
 	 << "\tFaces: " << nexus.totface
-	 << "\tPatches: " << nexus.index.size() << "\n\n";
+	 << "\tPatches: " << nexus.index.size() 
+	 << "\nChunk size " << nexus.chunk_size << endl;
+    
   }
   
   //determine if we must proceed:
@@ -239,15 +256,24 @@ int main(int argc, char *argv[]) {
   unsigned int signature = nexus.signature;
   signature |= add;
   signature &= ~remove;
+  if(compress) signature |= NXS_COMPRESSED;
+  if(uncompress) signature &= ~NXS_COMPRESSED;
 
   if(!output.size()) output = input + getSuffix(signature);
 
+  cerr << "Writing to nexus: " << output << endl;
+
   Nexus out;
-  if(!out.Create(output, (Signature)signature)) {
+  out.patches.SetRamBufferSize(ram_size);
+  if(!chunk_size)
+    chunk_size = nexus.patches.chunk_size;
+
+  if(!out.Create(output, (Signature)signature, chunk_size)) {
     cerr << "Could not open output: " << output << endl;
     return -1;
   }
 
+  
   for(unsigned int patch = 0; patch < nexus.index.size(); patch++) {
     Nexus::PatchInfo &src_entry = nexus.index[patch];
     Patch src_patch = nexus.GetPatch(patch);

apps/nexus/patch.cpp

@@ -1,7 +1,11 @@
 #include "patch.h"
-
+#include <lzo1x.h>
+#include <iostream>
+using namespace std;
 using namespace nxs;
 
+static double wrkmem[LZO1X_999_MEM_COMPRESS/sizeof(double) +1];
+
 void pad(unsigned int &size) {
   while(size&0x3) size++;
 }
@@ -46,27 +50,6 @@ void Patch::Init(Signature signature,
     dstart = tstart + nv;
   else 
     dstart = tstart;
-
-  /*  cstart = nv * sizeof(float) * 3;
-  if(signature & NXS_COLORS)
-    nstart = cstart + nv * sizeof(unsigned int);
-  else 
-    nstart = cstart;
-
-  if(signature & NXS_NORMALS_SHORT)
-    tstart = nstart + nv * sizeof(short) * 4;
-  else if(signature & NXS_NORMALS_FLOAT)
-    tstart = nstart + nv * sizeof(float) * 3;
-  else 
-    tstart = nstart;
-
-  if(signature & NXS_TEXTURES_SHORT)
-    dstart = tstart + nv * sizeof(short) * 2;
-  else if(signature & NXS_TEXTURES_FLOAT)
-    dstart = tstart + nv * sizeof(float) * 2;
-  else 
-  dstart = tstart;*/
-
 }
 
 unsigned int Patch::ChunkSize(Signature signature, 
@@ -126,3 +109,47 @@ unsigned int Patch::ByteSize(Signature signature,
   
   return size;
 }
+
+
+char *Patch::Compress(unsigned int ram_size, unsigned int &size) {
+  //TODO use OVERLAP and test speed
+  //TODO fill chunk padding with zeroes?
+  //TODO compress only used memory!
+  size = ram_size + ram_size/64 + 23;
+  char *buffer = new char[size];
+  lzo1x_1_compress(((unsigned char *)start), ram_size,
+		     (unsigned char *)buffer + sizeof(int), &size,
+		   (char *)wrkmem);
+  
+  *(int *)buffer = size;
+  size += sizeof(int);
+  
+
+  //  memcpy(buffer, start, ram_size);
+  //  size = ram_size;
+  //TODO optimize!
+  //      lzo1x_optimize((unsigned char *)entry.patch->start,
+  //			 entry.ram_size * chunk_size,
+  //			 compressed, &compressed_size, NULL);
+
+
+  return buffer;
+}
+
+void Patch::Decompress(unsigned int ram_size, void *src, unsigned int src_sz) {
+		     
+  unsigned int size = *(int *)src;
+  assert(size < src_sz + sizeof(int));
+  unsigned int dst_size = ram_size;
+  
+  //  memcpy(start, src, ram_size);
+  int ret = lzo1x_decompress_safe(((unsigned char *)src) + sizeof(int), size,
+				  (unsigned char *)start, &dst_size, 0);
+  if(ret != 0) {
+    cerr << "Ret from decompress: " << ret << endl;
+    exit(-1);
+  }
+  assert(dst_size == ram_size);
+  //TODO add 3 to start... so we can use asm_fast decompressor  
+}
+

apps/nexus/patch.h

@@ -50,12 +50,14 @@ class Patch {
 			       unsigned short nvert, 
 			       unsigned short nface);
   
+  char *Compress(unsigned int ram_size, unsigned int &size);
+  void Decompress(unsigned int ram_size, void *src, unsigned int src_sz);
 		  
-  char *start;
 
   unsigned short nv;
   unsigned short nf;
 
+  char *start;
   float *vstart;
   //these offset are from vstart!
   unsigned short cstart;

apps/nexus/patchserver.cpp

@@ -1,5 +1,4 @@
 #include "patchserver.h"
-
 #include <iostream>
 #include <algorithm>
 
@@ -8,6 +7,12 @@ using namespace nxs;
 
 //TODO support compression!
 
+void outbuffer(unsigned char *b, unsigned int s) {
+  for(unsigned int i = 0; i < s; i++)
+    cerr << (unsigned int)b[i];
+  cerr << endl;
+}
+
 
 bool PatchServer::Create(const std::string &filename, 
 			 Signature sig, 
@@ -47,8 +52,8 @@ bool PatchServer::ReadEntries(FILE *fp) {
   for(unsigned int i = 0; i < n; i++) {
     patches[i].patch = NULL;
     fread(&(patches[i].patch_start), 1, sizeof(unsigned int), fp);
-    fread(&(patches[i].patch_size),  1, sizeof(unsigned short), fp);
+    fread(&(patches[i].ram_size),  1, sizeof(unsigned short), fp);
-    fread(&(patches[i].ram_used),    1, sizeof(unsigned short), fp);
+    fread(&(patches[i].disk_size),    1, sizeof(unsigned short), fp);
     patches[i].lru_pos = 0xffffffff;
   }
   return true;
@@ -59,8 +64,8 @@ bool PatchServer::WriteEntries(FILE *fp) {
   fwrite(&n, 1, sizeof(int), fp);
   for(unsigned int i = 0; i < patches.size(); i++) {
     fwrite(&(patches[i].patch_start), 1, sizeof(unsigned int), fp);
-    fwrite(&(patches[i].patch_size),  1, sizeof(unsigned short), fp);
+    fwrite(&(patches[i].ram_size),  1, sizeof(unsigned short), fp);
-    fwrite(&(patches[i].ram_used),    1, sizeof(unsigned short), fp);
+    fwrite(&(patches[i].disk_size),    1, sizeof(unsigned short), fp);
   }
   return true;
 }
@@ -68,37 +73,64 @@ bool PatchServer::WriteEntries(FILE *fp) {
 void PatchServer::AddPatch(unsigned short nvert, unsigned short nface) {
   PatchEntry entry;
   entry.patch = NULL;
-  entry.patch_start = Length()/chunk_size;
+  entry.ram_size = Patch::ChunkSize(signature, nvert, nface, chunk_size);
-  entry.patch_size = Patch::ChunkSize(signature, nvert, nface, chunk_size);
+
-  entry.ram_used = entry.patch_size;
+  //if compressed we do not allocate space now (how much anyway?)
+  //  if((signature & NXS_COMPRESSED) != 0) {
+    entry.disk_size = 0xffff;
+    entry.patch_start = 0xffffffff;
+    /*  }  else {
+    entry.disk_size = entry.ram_size;
+    entry.patch_start = Length()/chunk_size;
+    Redim(Length() + entry.disk_size * chunk_size);
+    }*/
   entry.lru_pos = 0xffffffff;
   patches.push_back(entry);
   
-  Redim(Length() + entry.patch_size * chunk_size);
+
 }
 
 Patch &PatchServer::GetPatch(unsigned int idx, 
 			     unsigned short nvert, unsigned short nface,
 			     bool flush) {
-
   assert(idx < patches.size());
   PatchEntry &entry = patches[idx];
 
-  if(entry.patch) { 
+  if(entry.patch) {  //already on buffer
     assert(entry.lru_pos < lru.size());
     assert(lru[entry.lru_pos].patch == idx);
     lru[entry.lru_pos].frame = frame++;
+
   } else {
-    SetPosition(entry.patch_start * chunk_size);
 
-    assert(entry.patch_size != 0);
+    assert(entry.lru_pos == 0xffffffff);
-    char *start = new char[entry.patch_size * chunk_size];
+    if(flush) Flush();
-    ReadBuffer(start, entry.patch_size * chunk_size);
+
+
+    char *ram = new char[entry.ram_size * chunk_size];
+    entry.patch = new Patch(signature, ram, nvert, nface);
+    
+    if(entry.patch_start != 0xffffffff) { //was allocated.
+      assert(entry.disk_size != 0xffff);
+
+      SetPosition(entry.patch_start * chunk_size);
+    
+      if((signature & NXS_COMPRESSED) == 0) { //not compressed
+	ReadBuffer(ram, entry.disk_size * chunk_size);
+      } else {
+
+	unsigned char *disk = new unsigned char[entry.disk_size * chunk_size];
+	ReadBuffer(disk, entry.disk_size * chunk_size);
+
+	entry.patch->Decompress(entry.ram_size * chunk_size, 
+				disk, entry.disk_size * chunk_size);
+	delete []disk;
+      } 
+    }
 
-    entry.patch = new Patch(signature, start, nvert, nface);
     entry.lru_pos = lru.size();
     lru.push_back(PTime(idx, frame++));
-    ram_used += entry.ram_used;
+    ram_used += entry.ram_size;
   }
   
   //avoid frame overflow!
@@ -112,27 +144,23 @@ Patch &PatchServer::GetPatch(unsigned int idx,
     for(unsigned int i = 0; i < lru.size(); i++)
       patches[lru[i].patch].lru_pos = i;
   }
-
-  if(flush && ram_used > ram_size * 1.1)
-    Flush();
-
   return *(entry.patch); 
 }
 
 void PatchServer::Flush() {
-  cerr << "FLUSHING\n\n\n n";
+
-  cerr << "ram_size: " << ram_size << endl;
+  if(ram_used < ram_size * 1.5) return;
-  cerr << "ram_used: " << ram_used << endl;
+
   make_heap(lru.begin(), lru.end());
+  for(unsigned int i = 0; i < lru.size(); i++)
+    patches[lru[i].patch].lru_pos = i;
+
   while(ram_used > ram_size) {
     pop_heap(lru.begin(), lru.end());
     PTime &ptime = lru.back();
-
     Flush(ptime.patch);
-
     lru.pop_back();
   }
-  make_heap(lru.begin(), lru.end());
   for(unsigned int i = 0; i < lru.size(); i++)
     patches[lru[i].patch].lru_pos = i;
 }
@@ -146,19 +174,53 @@ void PatchServer::FlushAll() {
   lru.clear();
 }
 
-
 void PatchServer::Flush(unsigned int patch) {
-
   PatchEntry &entry = patches[patch];
   assert(entry.patch);
 
   if(!readonly) { //write back patch
-    SetPosition(entry.patch_start * chunk_size);
+
-    WriteBuffer(entry.patch->start, entry.patch_size * chunk_size);
+
+    if((signature & NXS_COMPRESSED)) {
+      unsigned int compressed_size;
+      char *compressed = entry.patch->Compress(entry.ram_size * chunk_size,
+					       compressed_size);
+      if(entry.disk_size == 0xffff) {//allocate space 
+	assert(entry.patch_start == 0xffffffff);
+	entry.disk_size = (unsigned int)((compressed_size-1)/chunk_size) + 1;
+	entry.patch_start = Length()/chunk_size;
+	Redim(Length() + entry.disk_size * chunk_size);
+      } else {
+	cerr << "OOOOPSPPPS not supported!" << endl;
+	exit(-1);
+      }
+      cerr << "Ram size: " << entry.ram_size << endl;
+      cerr << "Disk size: " << entry.disk_size << endl;
+      SetPosition(entry.patch_start * chunk_size);
+      WriteBuffer(compressed, entry.disk_size * chunk_size);
+      delete []compressed;
+    } else {
+      if(entry.disk_size == 0xffff) {
+	entry.disk_size = entry.ram_size;
+	entry.patch_start = Length()/chunk_size;
+	Redim(Length() + entry.disk_size * chunk_size);
+      }
+      SetPosition(entry.patch_start * chunk_size);
+      WriteBuffer(entry.patch->start, entry.disk_size * chunk_size);
+    }
+    /*    FILE *fo = fopen("tmp", "wb+");
+    fwrite(entry.patch->start, 1, entry.disk_size * chunk_size, fo);
+    fclose(fo);
+    exit(0);*/
   }
+
   delete [](entry.patch->start);
   delete entry.patch;
   entry.patch = NULL;
   entry.lru_pos = 0xffffffff;
-  ram_used -= entry.ram_used;
+  ram_used -= entry.ram_size;
+}
+
+void PatchServer::SetRamBufferSize(unsigned int r_buffer) {
+  ram_size = (unsigned int)(r_buffer/chunk_size) + 1;
 }

apps/nexus/patchserver.h

@@ -11,8 +11,8 @@ namespace nxs {
 struct PatchEntry { 
   Patch *patch;
   unsigned int patch_start;  //granularita' Chunk
-  unsigned short patch_size;  //in chunks
+  unsigned short ram_size;  //in chunks 
-  unsigned short ram_used;  // in chunks (used when compressed)
+  unsigned short disk_size;  // in chunks (used when compressed)
   unsigned int lru_pos;
 };
 
@@ -25,7 +25,7 @@ class PatchServer: public File {
     PTime(unsigned int p = 0xffffffff, unsigned int f = 0xffffffff):
 	 patch(p), frame(f) {}
 
-    bool operator<(const PTime &p) const { return frame < p.frame; }
+    bool operator<(const PTime &p) const { return frame > p.frame; }
   };
 
 
@@ -54,6 +54,7 @@ class PatchServer: public File {
   void Flush();
   void FlushAll();
   void Flush(unsigned int patch);
+  void SetRamBufferSize(unsigned int ram_buffer);
 
   std::vector<PatchEntry> patches;
   std::vector<PTime> lru;

apps/nexus/voronoichain.cpp

@@ -24,6 +24,9 @@
   History
 
 $Log: not supported by cvs2svn $
+Revision 1.10  2004/10/09 14:46:47  ponchio
+Windows porting small changes.
+
 Revision 1.9  2004/10/08 15:12:04  ponchio
 Working version (maybe)
 
@@ -132,7 +135,7 @@ void VoronoiChain::Init(Crude &crude, float scaling, int steps) {
     if(fcount[i] > min_size)
       seeds.push_back(fine[i]);
   }
-  swap((std::vector<Seed>)fine, seeds);
+  swap(fine, seeds);
   if(fine.size() == 0) fine.push_back(Point3f(0,0,0));
   fine.Init();
 
@@ -170,7 +173,7 @@ void VoronoiChain::Init(Crude &crude, float scaling, int steps) {
     if(ccount[i] > (int)min_size)
       seeds.push_back(coarse[i]);
   }
-  swap((std::vector<Seed>)coarse, seeds);
+  swap(coarse, seeds);
   if(coarse.size() == 0) coarse.push_back(Point3f(0,0,0));
   coarse.Init();
 
@@ -404,7 +407,7 @@ void VoronoiChain::BuildLevel(Nexus &nexus, unsigned int offset,
     if(ccount[i] > (int)min_size)
       seeds.push_back(coarse[i]);
   }
-  swap((vector<Seed>)coarse, seeds);
+  swap(coarse, seeds);
   if(coarse.size() == 0) coarse.push_back(Point3f(0,0,0));
   coarse.Init();
 

apps/nexus/voronoinxs.cpp

@@ -24,6 +24,9 @@
   History
 
 $Log: not supported by cvs2svn $
+Revision 1.13  2004/10/09 17:32:25  ponchio
+Ram buffer option added (last)
+
 Revision 1.12  2004/10/09 17:29:04  ponchio
 Ram buffer option added (again)
 
@@ -131,9 +134,10 @@ int main(int argc, char *argv[]) {
   unsigned int max_level = 0xffffffff;
   float scaling = 0.5;
   unsigned int ram_buffer = 128000000;
+  unsigned int chunk_size = 1024;
 
   int option;
-  while((option = getopt(argc, argv, "f:t:l:s:d:ro:b:")) != EOF) {
+  while((option = getopt(argc, argv, "f:t:l:s:d:ro:b:c:")) != EOF) {
     switch(option) {
     case 'f': patch_size = atoi(optarg);
       if(patch_size == 0 || patch_size > 32000) {
@@ -171,7 +175,18 @@ int main(int argc, char *argv[]) {
       break;
     case 'r': stop_after_remap = true; break;
     case 'o': optimization_steps = atoi(optarg); break;
-    case 'b': ram_buffer = atoi(optarg); break;
+    case 'b': ram_buffer = atoi(optarg); 
+      if(ram_buffer == 0) {
+	cerr << "Invalid ram buffer: " << optarg << endl;
+	return -1;
+      }
+      break;
+    case 'c': chunk_size = atoi(optarg);
+      if(chunk_size == 0) {
+	cerr << "Invalid chunk size: " << optarg << endl;
+	return -1;
+      }
+      break;
     default: cerr << "Unknown option: " << (char)option << endl;
       return -1;
     }
@@ -206,11 +221,12 @@ int main(int argc, char *argv[]) {
   string output = argv[optind+1];
 
   Nexus nexus;
-  if(!nexus.Create(output, NXS_FACES)) {
+  nexus.patches.SetRamBufferSize(ram_buffer);
+  if(!nexus.Create(output, NXS_FACES, chunk_size)) {
     cerr << "Could not create nexus output: " << output << endl;
     return -1;
   }
-  nexus.patches.ram_size = ram_buffer;
+
 
   if(patch_threshold == 0xffffffff)
     patch_threshold = patch_size/4;