Cut plane with big option fix

src/FVMAdapt/get_node_remap.loci

@@ -53,27 +53,20 @@ public:
     disable_threading() ;
   }
   void compute(const sequence &seq) {
-   if(Loci::MPI_processes == 1) {
+   fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ;
+   if(df == 0) {
 
      for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){
        fileNumX[*si] = *si ;
-       // if(*si < 0) cout << "negative file Number " << *si << endl; 
      }
-     return;
+   } else {
+     Map l2f ;
+     l2f = df->l2f.Rep() ;
+
+     for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){
+       fileNumX[*si] = l2f[*si] ; 
+     }
    }
-   fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ;
-    Map l2g ;
-    l2g = df->l2g.Rep() ;
-    store<unsigned char> key_domain ;
-    key_domain = df->key_domain.Rep() ;
-    //    dMap g2f ;
-    //    g2f = df->g2f.Rep() ;
-
-    for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){
-      int kd = key_domain[*si] ;
-      fileNumX[*si] = df->g2fv[kd][l2g[*si]] ;
-      //if(fileNumX[*si] < 0) cout << "negative file Number " << fileNumX[*si] << endl; 
-    }
   }
 
 } ;

src/FVMAdapt/library/face.cc

@@ -554,7 +554,7 @@ void Face::resplit(const std::vector<char>& facePlan,
 
       default:
         cerr <<"WARNING: illegal splitcode in function Face::resplit()" << endl;
-
+        fatal(currentCode>1) ;
         break;
       }
 
@@ -612,7 +612,7 @@ void Face::resplit(const std::vector<char>& facePlan,
 
       default:
         cerr <<"WARNING: illegal splitcode in function Face::resplit()" << endl;
-
+        fatal(currentCode>1) ;
         break;
       }
 
@@ -670,6 +670,7 @@ void Face::resplit(const std::vector<char>& facePlan,
 
       default:
         cerr <<"WARNING: illegal splitcode in function Face::reSplit(char orientCode)" << endl;
+        fatal(currentCode>1) ;
         break;
       }
 
@@ -726,6 +727,7 @@ void  Face::empty_resplit(const std::vector<char>& facePlan,
 
       default:
         cerr <<"WARNING: illegal splitcode in function Face::empty_resplit()" << endl;
+        fatal(currentCode>1) ;
         break;
       }
 
@@ -784,6 +786,7 @@ int  Face::empty_resplit(const std::vector<char>& facePlan){
 
       default:
         cerr <<"WARNING: illegal splitcode in function Face::empty_resplit()" << endl;
+        fatal(currentCode>1) ;
 
         break;
       }

src/FVMAdapt/library/write_vog.cc

@@ -132,11 +132,10 @@ namespace Loci {
       DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ;
       return c2pg.Rep() ;
     }
-    // Now get the maps needed to translate from global to file
-    dMap g2f ;
-    g2f = dist->g2f.Rep() ;
     Map l2g ;
     l2g = dist->l2g.Rep() ;
+    Map l2f ;
+    l2f = dist->l2f.Rep() ;
     constraint geom_cells ;
     geom_cells = facts.get_fact("geom_cells") ;
     dom = geom_cells.Rep()->domain() ;
@@ -149,7 +148,7 @@ namespace Loci {
     int xfol = std::numeric_limits<int>::lowest() ;
     FORALL(dom,ii) {
       int g = l2g[ii] ;
-      int f = g2f[g] ;
+      int f = l2f[ii] ;
       f2g[cnt++] = pair<int,int>(f,g) ;
       mfol = min(mfol,f) ;
       xfol = max(xfol,f) ;
@@ -1567,10 +1566,11 @@ namespace Loci{
   // MPI Communicator(comm)
   storeRepP Global2FileOrder(storeRepP sp, entitySet dom, int &offset,
                              fact_db::distribute_infoP dist, MPI_Comm comm) {
-
+
+    int keyspace = sp->getDomainKeySpace() ;
     // Now get global to file numbering
     dMap g2f ;
-    g2f = dist->g2fv[0].Rep() ; // FIX THIS
+    g2f = dist->g2fv[keyspace].Rep() ; 
 
     // Compute map from local numbering to file numbering
     Map newnum ;
@@ -2055,7 +2055,7 @@ namespace Loci{
     // update remap from global to file numbering for faces after sorting
     fact_db::distribute_infoP df = facts.get_distribute_info() ;
     dMap g2f ;
-    g2f = df->g2fv[0].Rep() ; // FIX THIS
+    g2f = df->g2fv[fk].Rep() ; 
 
     int cells_base=local_cells[0].Min() ;
     for(size_t i=0;i<volTags.size();++i) {

src/FVMMod/misc.loci

@@ -58,17 +58,11 @@ namespace Loci {
   prelude {
     fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ;
 
-    if(Loci::MPI_processes > 1) {
-      Map l2g ;
-      l2g = df->l2g.Rep() ;
-      store<unsigned char> key_domain ;
-      key_domain = df->key_domain.Rep() ;
-      //        dMap g2f ;
-      //        g2f = df->g2f.Rep() ;
-
+    if(df != 0) {
+      Map l2f ;
+      l2f = df->l2f.Rep() ;
       for(sequence::const_iterator si=seq.begin();si!= seq.end();++si) {
-        int kd = key_domain[*si] ;
-        $fileNumber(X)[*si] = df->g2fv[kd][l2g[*si]] ;
+        $fileNumber(X)[*si] = l2f[*si] ; 
       }
     } else {
       for(sequence::const_iterator si=seq.begin();si!= seq.end();++si)

src/FVMtools/vogCheck.loci

@@ -25,12 +25,16 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <vector>
+#include "distribute_io.h"
 #include "docreport.h"
 
 $include "FVM.lh"
 
 namespace Loci {
   void create_ci_map(fact_db &facts) ;
+  extern int factdb_allocated_base ;
+  extern bool useDomainKeySpaces ;
+  extern unsigned long readAttributeLong(hid_t group, const char *name) ;
 }
 
 using Loci::real_t ;
@@ -43,6 +47,42 @@ using std::ostringstream ;
 using std::vector ;
 using Loci::variable ;
 using Loci::variableSet ;
+bool checkVOGFileSize(string filename) {
+  hid_t file_id = 0 ;
+  file_id = Loci::readVOGOpen(filename.c_str()) ;
+  int failure = 0 ;
+  if(Loci::MPI_rank == 0 && file_id <= 0) {
+    failure = 1 ;
+  }
+  int fail_state = 0 ;
+  MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ;
+  if(fail_state != 0) {
+    return false ;
+  }
+  bool bigfile = false ;
+  if(Loci::use_parallel_io || Loci::MPI_rank == 0) {
+    hid_t file_info = H5Gopen(file_id,"file_info",H5P_DEFAULT) ;
+    long long numCells = Loci::readAttributeLong(file_info,"numCells") ;
+    long long numFaces = Loci::readAttributeLong(file_info,"numFaces") ;
+    long long numNodes = Loci::readAttributeLong(file_info,"numNodes") ;
+
+    if(Loci::MPI_rank == 0) {
+      long long totalSize = numCells+numNodes+numFaces ;
+
+      cout << "File: " << filename << " reading " << totalSize << " entities."
+           << endl ;
+      if(totalSize+2048 > std::numeric_limits<int>::max()) {
+        bigfile = true ;
+        cout << "WARNING:: Grid file will need --big flag to process." << endl ;
+      }
+    }
+
+    H5Gclose(file_info) ;
+  }
+  Loci::hdf5CloseFile(file_id) ;
+  return bigfile ;
+}
+
 int main(int ac, char *av[]) {
   Loci::Init(&ac, &av) ;
   // default query 
@@ -120,6 +160,7 @@ int main(int ac, char *av[]) {
     exit(0) ;
   }
 
+  bool bigfile = false ;
   if(ac > 1) {
     string tmp = av[1] ;
 
@@ -135,13 +176,19 @@ int main(int ac, char *av[]) {
     string filename = string(av[1]) ;
     if(!has_dot)
       filename += ".vog" ;
+
+    Loci::factdb_allocated_base = std::numeric_limits<int>::min()+2048 ;
+    Loci::useDomainKeySpaces = true ;
+
+
     param<std::string> modelName ;
 
     *modelName = basename ;
     facts.create_fact("modelName",modelName) ;
     if(Loci::MPI_rank==0) {
       cout << "Reading: '" << filename <<"' ..." <<  endl  ;
     }
+    bigfile = checkVOGFileSize(filename);
     if(!Loci::setupFVMGrid(facts,filename)) {
       if(Loci::MPI_rank==0) {
         cerr << "Reading grid file '" << filename <<"' failed in grid reader!"
@@ -172,6 +219,9 @@ int main(int ac, char *av[]) {
     }
 
   }
+  if(Loci::MPI_rank == 0 && bigfile) {
+    cout << "-- NOTE: This is a large mesh, use --big to process." << endl;
+  }
   Loci::Finalize() ;
 }
 

src/System/FVMGridReader.cc

@@ -3022,12 +3022,20 @@ namespace Loci {
 
     vector<int> node_alloc(newnodes) ;
     int i=0;
+    int minNode = std::numeric_limits<int>::max() ;
+    for(int p=0;p<MPI_processes;++p) {
+      if(node_ptn_t[p] != EMPTY)
+        minNode = min(minNode,node_ptn_t[p].Min()) ;
+    }
+    int nodeOffset = minNode ;
+    MPI_Allreduce(&minNode,&nodeOffset,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ;
     for(int p=0;p<MPI_processes;++p) {
       FORALL(node_ptn_t[p], ni) {
-        node_alloc[i++] = ni ;
+        node_alloc[i++] = ni-nodeOffset ;
       } ENDFORALL;
     }
 
+
     entitySet nodes = facts.get_distributed_alloc(node_alloc,0).first ;// FIX THIS
     node_alloc.resize(0) ;
 

src/System/FVMStuff.cc

@@ -1396,7 +1396,7 @@ namespace Loci{
 
     double a = val[edge2node[e][0]] ;
     double b = val[edge2node[e][1]] ;
-    return ((b)/(b - a)) ;
+    return ((b)/(b - a + 1e-30)) ;
   }
 
 
@@ -3913,14 +3913,15 @@ namespace Loci{
     // Allocate entities for new edges
     int num_edges = emap.size() ;
     int ek = facts.getKeyDomain("Edges") ;
-    if(!useDomainKeySpaces) {
+    //    if(!useDomainKeySpaces) {
+    //      ek = 0 ;
+    //    }
+    if(Loci::MPI_processes == 0)
       ek = 0 ;
-    }
     int fk = face2node.Rep()->getDomainKeySpace() ;
 
     entitySet edges = facts.get_distributed_alloc(num_edges,ek).first ;
 
-
     //create constraint edges
     constraint edges_tag;
     *edges_tag = edges;
@@ -4280,7 +4281,7 @@ namespace Loci{
       fact_db::distribute_infoP dist = facts.get_distribute_info() ;
 
       FORALL(global2file.domain(), ei){
-        dist->g2fv[0][ei] = global2file[ei][0] ;
+        dist->g2fv[ek][ei] = global2file[ei][0] ;
       }ENDFORALL;
 
     }

src/System/Initialize.cc

@@ -872,6 +872,12 @@ namespace Loci {
 	  factdb_allocated_base = std::numeric_limits<int>::min() + 2048 ;
 	  useDomainKeySpaces = true ;
 	  i++ ;
+	} else if(!(strcmp((*argv)[i],"--lilbig"))) {
+	  factdb_allocated_base = std::numeric_limits<int>::min() + 2048 ;
+	  i++ ;
+	} else if(!(strcmp((*argv)[i],"--biglil"))) {
+	  useDomainKeySpaces = true ;
+	  i++ ;
         } else if(!strcmp((*argv)[i],"--threads")) {
           // determine the number of threads to use.
           // ideally we would like to detect the available

src/System/scheduler.cc

@@ -1888,8 +1888,29 @@ bool operator <(const timingData &d) const {
         for(variableSet::const_iterator vi=target.begin();
             vi!=target.end();++vi) {
           storeRepP srp = local_facts.get_variable(*vi) ;
-          // the results are clearly intensional facts
+          int varkey = 0 ;
+          if(srp->RepType() != Loci::PARAMETER) {
+            // Find the key space
+            entitySet vdom = srp->domain() ;
+            vdom &= df->key_domain.domain() ;
+            // Determine keyspace of container
+            int key_max = -1 ;
+            int key_min = 1024 ;
+            FORALL(vdom,ii) {
+              int k = df->key_domain[ii] ;
+              key_max = max(key_max,k) ;
+              key_min = min(key_min,k) ;
+            } ENDFORALL ;
+            varkey = key_max ;
+            MPI_Allreduce(&key_max,&varkey,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ;
+            // Check that the container was in only a single keyspace
+            FATAL(key_max >= 0 && key_max != varkey) ;
+          }
+            // the results are clearly intensional facts
           facts.create_intensional_fact(*vi,srp->remap(l2g)) ;
+          // set key domain of the query variable
+          srp = facts.get_variable(*vi) ;
+          srp->setDomainKeySpace(varkey) ;
         }
       }else{
         for(variableSet::const_iterator vi=target.begin();

src/include/parSampleSort.h

@@ -61,7 +61,7 @@ namespace Loci {
     for(int i=0;i<p;++i) 
       bsizes[i] = psz + (i<rem?1:0) ;
 #ifdef DEBUG
-    int sum = 0 ;
+    long long sum = 0 ;
     for(int i=0;i<p;++i) 
       sum += bsizes[i] ;