Support for mfile (>4Gb)

apps/nexus/file.h

@@ -20,6 +20,18 @@ class File {
   File(): fp(NULL) {}
   ~File() { Close(); }   
 
+  File(const File &file) { 
+    fp = file.fp;
+    size = file.size;
+    readonly = file.readonly;
+    ((File &)file).fp = NULL;
+  }
+  bool operator=(const File &file) {
+    fp = file.fp;
+    size = file.size;
+    readonly = file.readonly;
+    ((File &)file).fp = NULL;
+  }
   bool Create(const std::string &filename);
   bool Load(const std::string &filename, bool readonly = false);
   void Close();

apps/nexus/mfile.cpp

@@ -0,0 +1,146 @@
+#include "mfile.h"
+#include <assert.h>
+#include <iostream>
+
+using namespace std;
+using namespace nxs;
+
+typedef unsigned long long u64;
+
+bool MFile::Create(const string &fname, unsigned int mxs) {
+  filename = fname;
+  files.clear();
+  size = 0;
+  readonly = false;
+  assert(mxs <= (1<<30));
+  max_size = mxs;
+  AddFile();
+  return true;
+}
+
+bool MFile::Load(const string &fname, bool ronly) {
+  filename = fname;
+  files.clear();
+  readonly = ronly;
+  max_size = (1<<30);
+  size = 0;
+  
+  while(1) {
+    string name = Name(files.size());
+    files.push_back(File());
+    File &file = files.back();
+    if(!file.Load(name, ronly)) {
+      files.pop_back();
+      break;
+    }
+    size += file.Length();
+  }
+  if(files.size() == 0) return false;
+  if(files.size() == 1) { 
+    assert(size <= max_size);
+  } else {
+    //SANITY TEST
+    for(unsigned int i = 0; i < files.size() -2; i++) {
+      if(files[i].Length() != files[i++].Length()) {
+	//"Inconsistent file size for some file.\n";
+	return false;
+      }
+      max_size = files[0].Length();
+    }
+  }
+  return true;
+}
+
+void MFile::Close() {
+  files.clear();
+}
+
+void MFile::Redim(unsigned long long sz) {
+  assert(!readonly);
+  if(sz > size) {
+    while(sz - size >= max_size) {
+      RedimLast(max_size);
+      AddFile();
+    }
+    RedimLast(sz - size);
+  } else {
+    while(size - files.back().Length() > sz)
+      RemoveFile();
+    assert(sz <= size);
+    RedimLast(files.back().Length() - (size - sz));
+  }    
+}
+
+void MFile::SetPosition(unsigned long long pos) {
+  assert(pos < size);
+  curr_fp = pos/(u64)max_size;
+  curr_pos = pos - (u64)max_size * (u64)curr_fp;
+  assert(curr_fp < files.size());
+  files[curr_fp].SetPosition(curr_pos);
+}
+
+void MFile::ReadBuffer(void *data, unsigned int sz) {
+  while(sz + curr_pos > max_size) {
+    unsigned int n = max_size - curr_pos;
+    files[curr_fp].ReadBuffer(data, n);
+    data = ((char *)data) + n;
+    sz -= n;
+    curr_fp++;
+    curr_pos = 0;
+    files[curr_fp].SetPosition(curr_pos);
+  }
+  files[curr_fp].ReadBuffer(data, sz);
+}
+
+void MFile::WriteBuffer(void *data, unsigned int sz) {
+  assert(!readonly);
+  while(sz + curr_pos > max_size) {
+    unsigned int n = max_size - curr_pos;
+    files[curr_fp].WriteBuffer(data, n);
+    data = ((char *)data) + n;
+    sz -= n;
+    curr_fp++;
+    curr_pos = 0;
+    files[curr_fp].SetPosition(curr_pos);
+  }
+  files[curr_fp].WriteBuffer(data, sz);
+}
+
+ void MFile::AddFile() {
+   string name = Name(files.size());
+   files.push_back(File());
+   File &file = files.back();
+   file.Create(name);
+ }
+
+ void MFile::RemoveFile() {
+   assert(files.size());
+
+   string name = Name(files.size()); 
+   File &file = files.back();
+   unsigned int last_size = file.Length();
+   files.pop_back();
+   size -= last_size;
+#ifdef WIN32
+   DeleteFile(name.c_str());
+#else
+   unlink(name.c_str());
+#endif
+ }
+
+void MFile::RedimLast(unsigned int sz) {
+  File &file = files.back();
+  unsigned int last_size = file.Length();
+  file.Redim(sz);
+  size += sz - last_size;
+}
+
+std::string MFile::Name(unsigned int n) {
+  char buffer[1024];
+  if(n == 0) 
+    sprintf(buffer, "%s", filename.c_str());
+  else
+    sprintf(buffer, "%s%d", filename.c_str(), n);
+  return string(buffer);
+}
+

apps/nexus/mfile.h

@@ -0,0 +1,52 @@
+#ifndef NXS_MFILE_H
+#define NXS_MFILE_H
+
+
+#include "file.h"
+#include <string>
+#include <vector>
+
+namespace nxs {
+
+
+class MFile {
+ public:
+
+  MFile() {}
+  ~MFile() { Close(); }   
+  
+  //max is  so default is 1 G
+  bool Create(const std::string &filename, 
+	      unsigned int max_file_size = (1<<30));
+  bool Load(const std::string &filename, bool readonly = false);
+  void Close();
+  
+  unsigned long long Length() { return size; }
+  void Redim(unsigned long long size);
+
+  void SetPosition(unsigned long long pos);
+  void ReadBuffer(void *data, unsigned int size);
+  void WriteBuffer(void *data, unsigned int size);
+
+  bool IsReadOnly() { return readonly; }
+
+ protected:
+  std::string filename;
+  std::vector<File> files;
+  unsigned int curr_pos;
+  unsigned int curr_fp;
+  unsigned long long size;
+  unsigned int max_size;
+  bool readonly;
+ private:
+  //all theese refer to the last in the fp.
+  void AddFile();
+  void RemoveFile();
+  void RedimLast(unsigned int sz);
+  unsigned int GetSize();
+  std::string Name(unsigned int n);
+};
+
+}
+
+#endif

apps/nexus/watch.cpp

@@ -24,6 +24,9 @@
   History
 
 $Log: not supported by cvs2svn $
+Revision 1.1  2004/10/19 17:20:24  ponchio
+renamed
+
 Revision 1.5  2004/10/19 17:16:52  ponchio
 Changed interface
 
@@ -41,7 +44,8 @@ First draft created.
 
 
 ****************************************************************************/
-#include "stopwatch.h"
+#include "watch.h"
+#include <stdio.h>
 
 #ifdef WIN32
 Watch::Watch(): elapsed(0) {
@@ -91,7 +95,7 @@ double Watch::Pause() {
 }
 
 void Watch::Continue() {
-  QueryPerformanceCounter(&tstart);
+  gettimeofday(&tstart, &tz); 
 }
 
 double Watch::Time() {
@@ -119,3 +123,27 @@ int Watch::Usec() {
   return ttime.tv_usec;
 #endif
 }
+
+void Report::Init(unsigned int t, float inter) {
+  watch.Start();
+  tot = t;
+  last = 0;
+  interval = inter;
+}
+
+void Report::Step(unsigned int count) {
+  if(count == 0) return;
+  float now = watch.Time();
+  if(now - last < interval) return;
+  //estimate final time
+  float tot_time = now * tot/(float)count;
+  printf("%d/%d\telapsed: %.1f\tremaining: %.1f\ttotal: %.1f\n",
+	 count, tot, now, tot_time - now, tot_time);
+  last = now;
+}
+
+void Report::Finish() {
+  float now = watch.Time();
+  printf("Tot: %.1f\tN: %d\tN/sec: %.1f\n",
+	 now, tot, tot/now);
+}

apps/nexus/watch.h

@@ -24,6 +24,9 @@
   History
 
 $Log: not supported by cvs2svn $
+Revision 1.1  2004/10/19 17:20:24  ponchio
+renamed
+
 Revision 1.4  2004/10/19 17:16:53  ponchio
 Changed interface
 
@@ -67,3 +70,16 @@ private:
 #endif
   double elapsed;
 };
+
+class Report {
+ public:
+  Report(unsigned int tot = 1, float inter = 5.0f) { Init(tot, inter); }
+  void Init(unsigned int tot, float inter = 5.0f);
+  void Step(unsigned int count);
+  void Finish();
+ private:
+  Watch watch;
+  int tot;
+  float last;
+  float interval;
+};