From 4d6098da12b8418b7f92be56a248e3e3748f1e7a Mon Sep 17 00:00:00 2001 From: "Edward A. Luke" Date: Sun, 31 May 2026 19:17:43 +0200 Subject: [PATCH 01/11] fixed issue with createEdges that caused cutplane features to fail wiht --big --- src/FVMAdapt/library/write_vog.cc | 4 +++- src/System/FVMStuff.cc | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/FVMAdapt/library/write_vog.cc b/src/FVMAdapt/library/write_vog.cc index ebf49ffc..8efa307e 100644 --- a/src/FVMAdapt/library/write_vog.cc +++ b/src/FVMAdapt/library/write_vog.cc @@ -134,7 +134,9 @@ namespace Loci { } // Now get the maps needed to translate from global to file dMap g2f ; - g2f = dist->g2f.Rep() ; + //g2f = dist->g2f.Rep() ; + // Hardwired, 0 is the cell key space + g2f = dist->g2fv[0].Rep() ; Map l2g ; l2g = dist->l2g.Rep() ; constraint geom_cells ; diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index b2553244..365222f8 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -3918,7 +3918,10 @@ namespace Loci{ } int fk = face2node.Rep()->getDomainKeySpace() ; + debugout << "ek = " << ek << endl ; entitySet edges = facts.get_distributed_alloc(num_edges,ek).first ; + debugout << "edges =" << edges << endl ; + // return ; //create constraint edges @@ -4280,7 +4283,7 @@ namespace Loci{ fact_db::distribute_infoP dist = facts.get_distribute_info() ; FORALL(global2file.domain(), ei){ - dist->g2fv[0][ei] = global2file[ei][0] ; + dist->g2fv[ek][ei] = global2file[ei][0] ; }ENDFORALL; } From bb39c5e1d90bda7a5545b9d69417222ac3cebf73 Mon Sep 17 00:00:00 2001 From: "Edward A. Luke" Date: Sun, 31 May 2026 19:26:21 +0200 Subject: [PATCH 02/11] removed some diagonostics --- src/System/FVMStuff.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index 365222f8..76c80941 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -3918,11 +3918,7 @@ namespace Loci{ } int fk = face2node.Rep()->getDomainKeySpace() ; - debugout << "ek = " << ek << endl ; entitySet edges = facts.get_distributed_alloc(num_edges,ek).first ; - debugout << "edges =" << edges << endl ; - // return ; - //create constraint edges constraint edges_tag; From 0ac6716246f73a56c00bb4218f648a59a14e871e Mon Sep 17 00:00:00 2001 From: Ed Luke Date: Mon, 8 Jun 2026 10:07:52 -0500 Subject: [PATCH 03/11] Changed it so node file numberings start at zero even with --big option --- src/System/FVMGridReader.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/System/FVMGridReader.cc b/src/System/FVMGridReader.cc index b0d974e5..8f3ba544 100644 --- a/src/System/FVMGridReader.cc +++ b/src/System/FVMGridReader.cc @@ -3022,12 +3022,20 @@ namespace Loci { vector node_alloc(newnodes) ; int i=0; + int minNode = std::numeric_limits::max() ; + for(int p=0;p Date: Mon, 8 Jun 2026 19:41:49 -0500 Subject: [PATCH 04/11] Changed to directly use l2f map instead of l2g->g2f approach --- src/FVMAdapt/get_node_remap.loci | 25 +++++++++---------------- src/FVMMod/misc.loci | 14 ++++---------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/FVMAdapt/get_node_remap.loci b/src/FVMAdapt/get_node_remap.loci index a9e89a5d..427589fa 100644 --- a/src/FVMAdapt/get_node_remap.loci +++ b/src/FVMAdapt/get_node_remap.loci @@ -53,27 +53,20 @@ public: disable_threading() ; } void compute(const sequence &seq) { - if(Loci::MPI_processes == 1) { + fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ; + if(df == 0) { for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){ fileNumX[*si] = *si ; - // if(*si < 0) cout << "negative file Number " << *si << endl; } - return; + } else { + Map l2f ; + l2f = df->l2f.Rep() ; + + for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){ + fileNumX[*si] = l2f[*si] ; + } } - fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ; - Map l2g ; - l2g = df->l2g.Rep() ; - store key_domain ; - key_domain = df->key_domain.Rep() ; - // dMap g2f ; - // g2f = df->g2f.Rep() ; - - for(sequence::const_iterator si=seq.begin();si!= seq.end();++si){ - int kd = key_domain[*si] ; - fileNumX[*si] = df->g2fv[kd][l2g[*si]] ; - //if(fileNumX[*si] < 0) cout << "negative file Number " << fileNumX[*si] << endl; - } } } ; diff --git a/src/FVMMod/misc.loci b/src/FVMMod/misc.loci index ccb4907a..8ba2e8a2 100644 --- a/src/FVMMod/misc.loci +++ b/src/FVMMod/misc.loci @@ -58,17 +58,11 @@ namespace Loci { prelude { fact_db::distribute_infoP df = Loci::exec_current_fact_db->get_distribute_info() ; - if(Loci::MPI_processes > 1) { - Map l2g ; - l2g = df->l2g.Rep() ; - store key_domain ; - key_domain = df->key_domain.Rep() ; - // dMap g2f ; - // g2f = df->g2f.Rep() ; - + if(df != 0) { + Map l2f ; + l2f = df->l2f.Rep() ; for(sequence::const_iterator si=seq.begin();si!= seq.end();++si) { - int kd = key_domain[*si] ; - $fileNumber(X)[*si] = df->g2fv[kd][l2g[*si]] ; + $fileNumber(X)[*si] = l2f[*si] ; } } else { for(sequence::const_iterator si=seq.begin();si!= seq.end();++si) From 6233475a2acaf341a15279772c432aad14c82742 Mon Sep 17 00:00:00 2001 From: Ed Luke Date: Mon, 8 Jun 2026 19:42:32 -0500 Subject: [PATCH 05/11] Fixed a few places where keyspaces weren't handled properly --- src/FVMAdapt/library/write_vog.cc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/FVMAdapt/library/write_vog.cc b/src/FVMAdapt/library/write_vog.cc index 8efa307e..5b58918d 100644 --- a/src/FVMAdapt/library/write_vog.cc +++ b/src/FVMAdapt/library/write_vog.cc @@ -132,13 +132,10 @@ namespace Loci { DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ; return c2pg.Rep() ; } - // Now get the maps needed to translate from global to file - dMap g2f ; - //g2f = dist->g2f.Rep() ; - // Hardwired, 0 is the cell key space - g2f = dist->g2fv[0].Rep() ; Map l2g ; l2g = dist->l2g.Rep() ; + Map l2f ; + l2f = dist->l2f.Rep() ; constraint geom_cells ; geom_cells = facts.get_fact("geom_cells") ; dom = geom_cells.Rep()->domain() ; @@ -151,7 +148,7 @@ namespace Loci { int xfol = std::numeric_limits::lowest() ; FORALL(dom,ii) { int g = l2g[ii] ; - int f = g2f[g] ; + int f = l2f[ii] ; f2g[cnt++] = pair(f,g) ; mfol = min(mfol,f) ; xfol = max(xfol,f) ; @@ -1569,10 +1566,11 @@ namespace Loci{ // MPI Communicator(comm) storeRepP Global2FileOrder(storeRepP sp, entitySet dom, int &offset, fact_db::distribute_infoP dist, MPI_Comm comm) { - + + int keyspace = sp->getDomainKeySpace() ; // Now get global to file numbering dMap g2f ; - g2f = dist->g2fv[0].Rep() ; // FIX THIS + g2f = dist->g2fv[keyspace].Rep() ; // Compute map from local numbering to file numbering Map newnum ; @@ -2057,7 +2055,7 @@ namespace Loci{ // update remap from global to file numbering for faces after sorting fact_db::distribute_infoP df = facts.get_distribute_info() ; dMap g2f ; - g2f = df->g2fv[0].Rep() ; // FIX THIS + g2f = df->g2fv[fk].Rep() ; int cells_base=local_cells[0].Min() ; for(size_t i=0;i Date: Mon, 8 Jun 2026 19:43:03 -0500 Subject: [PATCH 06/11] Added diagnostic code --- src/FVMAdapt/library/face.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/FVMAdapt/library/face.cc b/src/FVMAdapt/library/face.cc index e3b1fff0..91c20c68 100644 --- a/src/FVMAdapt/library/face.cc +++ b/src/FVMAdapt/library/face.cc @@ -554,7 +554,7 @@ void Face::resplit(const std::vector& facePlan, default: cerr <<"WARNING: illegal splitcode in function Face::resplit()" << endl; - + fatal(currentCode>1) ; break; } @@ -612,7 +612,7 @@ void Face::resplit(const std::vector& facePlan, default: cerr <<"WARNING: illegal splitcode in function Face::resplit()" << endl; - + fatal(currentCode>1) ; break; } @@ -670,6 +670,7 @@ void Face::resplit(const std::vector& facePlan, default: cerr <<"WARNING: illegal splitcode in function Face::reSplit(char orientCode)" << endl; + fatal(currentCode>1) ; break; } @@ -726,6 +727,7 @@ void Face::empty_resplit(const std::vector& facePlan, default: cerr <<"WARNING: illegal splitcode in function Face::empty_resplit()" << endl; + fatal(currentCode>1) ; break; } @@ -784,6 +786,7 @@ int Face::empty_resplit(const std::vector& facePlan){ default: cerr <<"WARNING: illegal splitcode in function Face::empty_resplit()" << endl; + fatal(currentCode>1) ; break; } From d094b616d45c6e191587dea208ff5d6d9d23afdc Mon Sep 17 00:00:00 2001 From: Ed Luke Date: Mon, 8 Jun 2026 19:43:49 -0500 Subject: [PATCH 07/11] Added code to add --lilbig and --biglil options --- src/System/Initialize.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/System/Initialize.cc b/src/System/Initialize.cc index b5f3e6ae..d5634667 100644 --- a/src/System/Initialize.cc +++ b/src/System/Initialize.cc @@ -868,6 +868,12 @@ namespace Loci { factdb_allocated_base = std::numeric_limits::min() + 2048 ; useDomainKeySpaces = true ; i++ ; + } else if(!(strcmp((*argv)[i],"--lilbig"))) { + factdb_allocated_base = std::numeric_limits::min() + 2048 ; + i++ ; + } else if(!(strcmp((*argv)[i],"--biglil"))) { + useDomainKeySpaces = true ; + i++ ; } else if(!strcmp((*argv)[i],"--threads")) { // determine the number of threads to use. // ideally we would like to detect the available From f26088a4ff2052fc0ef93d108c86eb4adb76563a Mon Sep 17 00:00:00 2001 From: Ed Luke Date: Tue, 9 Jun 2026 21:48:25 -0500 Subject: [PATCH 08/11] Updated edge creation to always use the Edge keyspace --- src/System/FVMStuff.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index 76c80941..5f2027a4 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -3913,9 +3913,9 @@ namespace Loci{ // Allocate entities for new edges int num_edges = emap.size() ; int ek = facts.getKeyDomain("Edges") ; - if(!useDomainKeySpaces) { - ek = 0 ; - } + // if(!useDomainKeySpaces) { + // ek = 0 ; + // } int fk = face2node.Rep()->getDomainKeySpace() ; entitySet edges = facts.get_distributed_alloc(num_edges,ek).first ; From 3b97c359fdaccad6a5e38721cbc8ceaf9dc80d8c Mon Sep 17 00:00:00 2001 From: Ed Luke Date: Tue, 9 Jun 2026 22:31:23 -0500 Subject: [PATCH 09/11] Added diagnostic for --big files, also changed vogCheck to always run in --big mode --- src/FVMtools/vogCheck.loci | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/FVMtools/vogCheck.loci b/src/FVMtools/vogCheck.loci index f1f5ff87..fb83ba24 100644 --- a/src/FVMtools/vogCheck.loci +++ b/src/FVMtools/vogCheck.loci @@ -25,12 +25,16 @@ #include #include #include +#include "distribute_io.h" #include "docreport.h" $include "FVM.lh" namespace Loci { void create_ci_map(fact_db &facts) ; + extern int factdb_allocated_base ; + extern bool useDomainKeySpaces ; + extern unsigned long readAttributeLong(hid_t group, const char *name) ; } using Loci::real_t ; @@ -43,6 +47,42 @@ using std::ostringstream ; using std::vector ; using Loci::variable ; using Loci::variableSet ; +bool checkVOGFileSize(string filename) { + hid_t file_id = 0 ; + file_id = Loci::readVOGOpen(filename.c_str()) ; + int failure = 0 ; + if(Loci::MPI_rank == 0 && file_id <= 0) { + failure = 1 ; + } + int fail_state = 0 ; + MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + if(fail_state != 0) { + return false ; + } + bool bigfile = false ; + if(Loci::use_parallel_io || Loci::MPI_rank == 0) { + hid_t file_info = H5Gopen(file_id,"file_info",H5P_DEFAULT) ; + long long numCells = Loci::readAttributeLong(file_info,"numCells") ; + long long numFaces = Loci::readAttributeLong(file_info,"numFaces") ; + long long numNodes = Loci::readAttributeLong(file_info,"numNodes") ; + + if(Loci::MPI_rank == 0) { + long long totalSize = numCells+numNodes+numFaces ; + + cout << "File: " << filename << " reading " << totalSize << " entities." + << endl ; + if(totalSize+2048 > std::numeric_limits::max()) { + bigfile = true ; + cout << "WARNING:: Grid file will need --big flag to process." << endl ; + } + } + + H5Gclose(file_info) ; + } + Loci::hdf5CloseFile(file_id) ; + return bigfile ; +} + int main(int ac, char *av[]) { Loci::Init(&ac, &av) ; // default query @@ -120,6 +160,7 @@ int main(int ac, char *av[]) { exit(0) ; } + bool bigfile = false ; if(ac > 1) { string tmp = av[1] ; @@ -135,6 +176,11 @@ int main(int ac, char *av[]) { string filename = string(av[1]) ; if(!has_dot) filename += ".vog" ; + + Loci::factdb_allocated_base = std::numeric_limits::min()+2048 ; + Loci::useDomainKeySpaces = true ; + + param modelName ; *modelName = basename ; @@ -142,6 +188,7 @@ int main(int ac, char *av[]) { if(Loci::MPI_rank==0) { cout << "Reading: '" << filename <<"' ..." << endl ; } + bigfile = checkVOGFileSize(filename); if(!Loci::setupFVMGrid(facts,filename)) { if(Loci::MPI_rank==0) { cerr << "Reading grid file '" << filename <<"' failed in grid reader!" @@ -172,6 +219,9 @@ int main(int ac, char *av[]) { } } + if(Loci::MPI_rank == 0 && bigfile) { + cout << "-- NOTE: This is a large mesh, use --big to process." << endl; + } Loci::Finalize() ; } From 8b271530056dd01140b8e36a73cc49cb134e3f82 Mon Sep 17 00:00:00 2001 From: Rob Harris Date: Thu, 11 Jun 2026 10:24:18 -0500 Subject: [PATCH 10/11] Added fix for potential divide-by-zero in cut plane edge weight calculation, which can occur when a==b=0. Added fix for int32 overflow in diagnostic DEBUG output in parSampleSort for large (--big, --lilbig) cases. --- src/System/FVMStuff.cc | 2 +- src/include/parSampleSort.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index 5f2027a4..b3aa6f60 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -1396,7 +1396,7 @@ namespace Loci{ double a = val[edge2node[e][0]] ; double b = val[edge2node[e][1]] ; - return ((b)/(b - a)) ; + return ((b)/(b - a + 1e-30)) ; } diff --git a/src/include/parSampleSort.h b/src/include/parSampleSort.h index 983ac2c8..4df30aeb 100644 --- a/src/include/parSampleSort.h +++ b/src/include/parSampleSort.h @@ -61,7 +61,7 @@ namespace Loci { for(int i=0;i Date: Sun, 28 Jun 2026 21:27:44 -0500 Subject: [PATCH 11/11] Fixed issue with queried facts not being in the correct key space. --- src/System/FVMStuff.cc | 2 ++ src/System/scheduler.cc | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index b3aa6f60..9509dbcf 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -3916,6 +3916,8 @@ namespace Loci{ // if(!useDomainKeySpaces) { // ek = 0 ; // } + if(Loci::MPI_processes == 0) + ek = 0 ; int fk = face2node.Rep()->getDomainKeySpace() ; entitySet edges = facts.get_distributed_alloc(num_edges,ek).first ; diff --git a/src/System/scheduler.cc b/src/System/scheduler.cc index a88e4f23..5cab731b 100644 --- a/src/System/scheduler.cc +++ b/src/System/scheduler.cc @@ -1888,8 +1888,29 @@ bool operator <(const timingData &d) const { for(variableSet::const_iterator vi=target.begin(); vi!=target.end();++vi) { storeRepP srp = local_facts.get_variable(*vi) ; - // the results are clearly intensional facts + int varkey = 0 ; + if(srp->RepType() != Loci::PARAMETER) { + // Find the key space + entitySet vdom = srp->domain() ; + vdom &= df->key_domain.domain() ; + // Determine keyspace of container + int key_max = -1 ; + int key_min = 1024 ; + FORALL(vdom,ii) { + int k = df->key_domain[ii] ; + key_max = max(key_max,k) ; + key_min = min(key_min,k) ; + } ENDFORALL ; + varkey = key_max ; + MPI_Allreduce(&key_max,&varkey,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + // Check that the container was in only a single keyspace + FATAL(key_max >= 0 && key_max != varkey) ; + } + // the results are clearly intensional facts facts.create_intensional_fact(*vi,srp->remap(l2g)) ; + // set key domain of the query variable + srp = facts.get_variable(*vi) ; + srp->setDomainKeySpace(varkey) ; } }else{ for(variableSet::const_iterator vi=target.begin();