From dfcdc0a394b80c1b5830b838e2147ec3c218791f Mon Sep 17 00:00:00 2001 From: Markus Battarbee Date: Mon, 11 May 2020 19:04:24 +0300 Subject: [PATCH 01/39] First attempt --- MAKE/Makefile.vorna_intel | 3 +- common.h | 4 +- datareduction/datareducer.cpp | 4 +- grid.cpp | 30 +++++++--- ioread.cpp | 2 +- iowrite.cpp | 2 +- vlasiator.cpp | 4 +- vlasovsolver/vlasovmover.cpp | 110 +++++++++++++++++++--------------- 8 files changed, 95 insertions(+), 64 deletions(-) diff --git a/MAKE/Makefile.vorna_intel b/MAKE/Makefile.vorna_intel index 0638b02c84..9393e00872 100644 --- a/MAKE/Makefile.vorna_intel +++ b/MAKE/Makefile.vorna_intel @@ -87,7 +87,8 @@ LDFLAGS += -Wl,-rpath=$(LIBRARY_PREFIX)/intel/phiprof/lib #header libraries INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ +#INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ +INC_DCCRG = -I/proj/markusb/libraries/dccrg/ INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass diff --git a/common.h b/common.h index 3426c24274..6461c41132 100644 --- a/common.h +++ b/common.h @@ -180,7 +180,9 @@ namespace CellParams { MAXRDT, /*!< maximum timestep allowed in ordinary space for this cell, * this is the max allowed timestep over all particle species.*/ MAXFDT, /*!< maximum timestep allowed in ordinary space by fieldsolver for this cell**/ - LBWEIGHTCOUNTER, /*!< Counter for storing compute time weights needed by the load balancing**/ + LBWEIGHTCOUNTERX, /*!< Counter for storing compute time weights needed by the load balancing**/ + LBWEIGHTCOUNTERY, /*!< Counter for storing compute time weights needed by the load balancing**/ + LBWEIGHTCOUNTERZ, /*!< Counter for storing compute time weights needed by the load balancing**/ ISCELLSAVINGF, /*!< Value telling whether a cell is saving its distribution function when partial f data is written out. */ FSGRID_RANK, /*!< Rank of this cell in the FsGrid cartesian communicator */ FSGRID_BOUNDARYTYPE, /*!< Boundary type of this cell, as stored in the fsGrid */ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 092e5e1a75..85b23644d2 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -355,7 +355,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight") { // Load balance metric for LB debugging - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTER,1)); + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); outputReducer->addMetadata(outputReducer->size()-1,"","","$\\mathrm{LB weight}$",""); continue; } @@ -970,7 +970,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti // continue; //} if(lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("loadbalance_weight",CellParams::LBWEIGHTCOUNTER,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); continue; } if(lowercase == "maxvdt" || lowercase == "maxdt_acceleration" || lowercase == "vg_maxdt_acceleration") { diff --git a/grid.cpp b/grid.cpp index 0bd98831ac..638a07b686 100644 --- a/grid.cpp +++ b/grid.cpp @@ -142,6 +142,12 @@ void initializeGrids( initializeStencils(mpiGrid); mpiGrid.set_partitioning_option("IMBALANCE_TOL", P::loadBalanceTolerance); + mpiGrid.set_partitioning_option("OBJ_WEIGHTS_COMPARABLE", "1"); + mpiGrid.set_partitioning_option("RCB_MULTICRITERIA_NORM", "3"); + /** RCB_MULTICRITERIA_NORM + Norm used in multicriteria algorithm; this determines how to balance the different weight constraints. Valid values are 1,2, and 3. Roughly, if the weights correspond to different phases, then the value 1 (1-norm) tries to minimize the total time (sum over all phases) while the value 3 (max-norm) attempts to minimize the worst imbalance in any phase. The 2-norm does something in between. Try a different value if you're not happy with the balance. **/ + mpiGrid.set_partitioning_option("OBJ_WEIGHT_DIM", "3"); + phiprof::start("Initial load-balancing"); if (myRank == MASTER_RANK) logFile << "(INIT): Starting initial load balance." << endl << writeVerbose; mpiGrid.balance_load(); @@ -233,7 +239,9 @@ void initializeGrids( phiprof::stop("Apply system boundary conditions state"); for (size_t i=0; iparameters[CellParams::LBWEIGHTCOUNTER] = 0; + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX] = 0; + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY] = 0; + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ] = 0; } for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTER] += mpiGrid[cells[i]]->get_number_of_velocity_blocks(popID); + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX] += mpiGrid[cells[i]]->get_number_of_velocity_blocks(popID); + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY] += mpiGrid[cells[i]]->get_number_of_velocity_blocks(popID); + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += mpiGrid[cells[i]]->get_number_of_velocity_blocks(popID); } } @@ -417,10 +427,14 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S //Set weight. If acceleration is enabled then we use the weight //counter which is updated in acceleration, otherwise we just //use the number of blocks. -// if (P::propagateVlasovAcceleration) - mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER]); -// else -// mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); + // if (P::propagateVlasovAcceleration) + std::vector lbthreeweight; + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX]); + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY]); + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ]); + mpiGrid.set_cell_weight_vector(cells[i], lbthreeweight); + // else + // mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); //reset counter //mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER] = 0.0; } diff --git a/ioread.cpp b/ioread.cpp index 7f99f8324f..f64cdfab2d 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -1159,7 +1159,7 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_dt2",CellParams::P_11_DT2,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_r",CellParams::P_11_R,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_v",CellParams::P_11_V,3,mpiGrid); } - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTER,1,mpiGrid); } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_v_dt",CellParams::MAXVDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_r_dt",CellParams::MAXRDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_fields_dt",CellParams::MAXFDT,1,mpiGrid); } diff --git a/iowrite.cpp b/iowrite.cpp index 1119b90f94..b6e78e2933 100644 --- a/iowrite.cpp +++ b/iowrite.cpp @@ -1349,7 +1349,7 @@ bool writeRestart(dccrg::Dccrg& mpiGrid, restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_dt2",CellParams::P_11_DT2,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_r",CellParams::P_11_R,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_v",CellParams::P_11_V,3)); - restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("LB_weight",CellParams::LBWEIGHTCOUNTER,1)); + restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("LB_weight",CellParams::LBWEIGHTCOUNTERX,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_v_dt",CellParams::MAXVDT,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_r_dt",CellParams::MAXRDT,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_fields_dt",CellParams::MAXFDT,1)); diff --git a/vlasiator.cpp b/vlasiator.cpp index 1c82c5a934..24371e262b 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -842,7 +842,9 @@ int main(int argn,char* args[]) { } #pragma omp parallel for for (size_t c=0; cget_cell_parameters()[CellParams::LBWEIGHTCOUNTER] = 0; + mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTERX] = 0; + mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTERY] = 0; + mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTERZ] = 0; } } diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index bae0fba323..2b49c4c373 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -71,7 +71,9 @@ void calculateSpatialTranslation( const vector& remoteTargetCellsx, const vector& remoteTargetCellsy, const vector& remoteTargetCellsz, - vector& nPencils, + vector& nPencilsX, + vector& nPencilsY, + vector& nPencilsZ, creal dt, const uint popID, Real &time @@ -85,10 +87,10 @@ void calculateSpatialTranslation( int myRank; MPI_Comm_rank(MPI_COMM_WORLD,&myRank); -// int bt=phiprof::initializeTimer("barrier-trans-pre-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + int bt=phiprof::initializeTimer("barrier-trans-pre-z","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); // ------------- SLICE - map dist function in Z --------------- // if(P::zcells_ini > 1){ @@ -98,25 +100,25 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_Z_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-z","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-z"); if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsz, 2, dt,popID); // map along z// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsz, nPencils, 2, dt,popID); // map along z// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsz, nPencilsZ, 2, dt,popID); // map along z// } phiprof::stop("compute-mapping-z"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-z","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-z","MPI"); phiprof::start("update_remote-z"); @@ -131,10 +133,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-pre-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-x","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); // ------------- SLICE - map dist function in X --------------- // if(P::xcells_ini > 1){ @@ -147,25 +149,25 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-x","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-x"); if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsx, 0,dt,popID); // map along x// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsx, nPencils, 0,dt,popID); // map along x// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsx, nPencilsX, 0,dt,popID); // map along x// } phiprof::stop("compute-mapping-x"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-x","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-x","MPI"); phiprof::start("update_remote-x"); @@ -180,10 +182,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-pre-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-y","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); // ------------- SLICE - map dist function in Y --------------- // if(P::ycells_ini > 1) { @@ -196,25 +198,25 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_Y_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-y","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-y"); if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsy, 1,dt,popID); // map along y// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencils, 1,dt,popID); // map along y// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencilsY, 1,dt,popID); // map along y// } phiprof::stop("compute-mapping-y"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-y","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-y","MPI"); phiprof::start("update_remote-y"); @@ -229,10 +231,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-post-trans","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + bt=phiprof::initializeTimer("barrier-trans-post-trans","Barriers","MPI"); + phiprof::start(bt); + MPI_Barrier(MPI_COMM_WORLD); + phiprof::stop(bt); // MPI_Barrier(MPI_COMM_WORLD); // bailout(true, "", __FILE__, __LINE__); @@ -263,7 +265,9 @@ void calculateSpatialTranslation( vector remoteTargetCellsz; vector local_propagated_cells; vector local_target_cells; - vector nPencils; + vector nPencilsX; + vector nPencilsY; + vector nPencilsZ; Real time=0.0; // If dt=0 we are either initializing or distribution functions are not translated. @@ -294,7 +298,9 @@ void calculateSpatialTranslation( if (P::prepareForRebalance == true && P::amrMaxSpatialRefLevel != 0) { // One more element to count the sums for (size_t c=0; cparameters[CellParams::LBWEIGHTCOUNTER] += time / localCells.size(); for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTER] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERX] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERY] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); } } } else { @@ -337,7 +347,9 @@ void calculateSpatialTranslation( for (uint popID=0; popIDget_number_of_velocity_blocks(popID); } - mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTER] += nPencils[c] * counter; + mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERX] += nPencilsX[c] * counter; + mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERY] += nPencilsY[c] * counter; + mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += nPencilsZ[c] * counter; // mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTER] += time / localCells.size(); } } From 3f97febe94910b8925a2950d4bc705ca57b9cc6e Mon Sep 17 00:00:00 2001 From: Markus Battarbee Date: Tue, 12 May 2020 14:36:16 +0300 Subject: [PATCH 02/39] Changed ioread to not complain about single LB value on readin --- ioread.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ioread.cpp b/ioread.cpp index f64cdfab2d..19eb708fda 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -1159,7 +1159,10 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_dt2",CellParams::P_11_DT2,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_r",CellParams::P_11_R,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_v",CellParams::P_11_V,3,mpiGrid); } - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,3,mpiGrid); } + // TODO this still reads old 1 LBW + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,1,mpiGrid); } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERY,1,mpiGrid); } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERZ,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_v_dt",CellParams::MAXVDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_r_dt",CellParams::MAXRDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_fields_dt",CellParams::MAXFDT,1,mpiGrid); } From 5dde4e7e39d88bb0cb25cd8d4596e2664cd06e5d Mon Sep 17 00:00:00 2001 From: Markus Battarbee Date: Tue, 28 Sep 2021 15:48:32 +0300 Subject: [PATCH 03/39] cleanup, support restarts with both vector and scalar LBweight values --- MAKE/Makefile.vorna_gcc | 2 +- datareduction/datareducer.cpp | 2 +- grid.cpp | 17 +++++++++-------- ioread.cpp | 36 +++++++++++++++++++++++++++++++---- vlasovsolver/vlasovmover.cpp | 19 +++++++----------- 5 files changed, 50 insertions(+), 26 deletions(-) diff --git a/MAKE/Makefile.vorna_gcc b/MAKE/Makefile.vorna_gcc index 503530342c..e2919bb831 100644 --- a/MAKE/Makefile.vorna_gcc +++ b/MAKE/Makefile.vorna_gcc @@ -89,7 +89,7 @@ INC_PAPI = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MP #header libraries INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ +INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg-threeweight/ INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/ INC_EIGEN = -I$(LIBRARY_PREFIX)/ -I$(LIBRARY_PREFIX)/Eigen diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index d73c727b9a..76719eb0f2 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -1008,7 +1008,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti // continue; //} if(lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTER,3)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); continue; } if(lowercase == "maxvdt" || lowercase == "maxdt_acceleration" || lowercase == "vg_maxdt_acceleration") { diff --git a/grid.cpp b/grid.cpp index 1894fef97f..6e555f9190 100644 --- a/grid.cpp +++ b/grid.cpp @@ -477,16 +477,17 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S //Set weight. If acceleration is enabled then we use the weight //counter which is updated in acceleration, otherwise we just //use the number of blocks. - // if (P::propagateVlasovAcceleration) std::vector lbthreeweight; - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX]); - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY]); - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ]); + if (P::propagateVlasovAcceleration) { + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX]); + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY]); + lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ]); + } else { + lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); + lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); + lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); + } mpiGrid.set_cell_weight_vector(cells[i], lbthreeweight); - // else - // mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); - //reset counter - //mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER] = 0.0; } phiprof::start("dccrg.initialize_balance_load"); mpiGrid.initialize_balance_load(true); diff --git a/ioread.cpp b/ioread.cpp index 076b40de68..e0b77dba1d 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -1011,6 +1011,28 @@ bool checkScalarParameter(vlsv::ParallelReader& file,const string& name,T correc } } +/*! +\brief read length of SpatialGrid vector variable +\param file The ParallelReader file to use +\param variableName Name of the variable to check +*/ +int readVectorSize(vlsv::ParallelReader& file, + const std::string& variableName){ + uint64_t arraySize; + uint64_t vectorSize; + vlsv::datatype::type dataType; + uint64_t byteSize; + list > attribs; + attribs.push_back(make_pair("name",variableName)); + attribs.push_back(make_pair("mesh","SpatialGrid")); + + if (file.getArrayInfo("VARIABLE",attribs,arraySize,vectorSize,dataType,byteSize) == false) { + logFile << "(RESTART) ERROR: Failed to read " << endl << write; + return -1; + } + return (int)vectorSize; +} + /*! \brief Read in state from a vlsv file in order to restart simulations \param mpiGrid Vlasiator's grid @@ -1201,10 +1223,16 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_dt2",CellParams::P_11_DT2,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_r",CellParams::P_11_R,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_v",CellParams::P_11_V,3,mpiGrid); } - // TODO this still reads old 1 LBW - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,1,mpiGrid); } - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERY,1,mpiGrid); } - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERZ,1,mpiGrid); } + if(success) { // Read 1 or 3 length LBweights + int LBlength = readVectorSize(file,string("LB_weight")); + if (LBlength==3) { + success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,3,mpiGrid); + } else { + success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,1,mpiGrid); + success=success && readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERY,1,mpiGrid); + success=success && readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERZ,1,mpiGrid); + } + } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_v_dt",CellParams::MAXVDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_r_dt",CellParams::MAXRDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_fields_dt",CellParams::MAXFDT,1,mpiGrid); } diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index ec836051a4..a240825563 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -213,7 +213,7 @@ void calculateSpatialTranslation( if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsy, 1,dt,popID); // map along y// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencilsY, 1,dt,popID); // map along y// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencilsY, 1,dt,popID); // map along y// } phiprof::stop("compute-mapping-y"); time += MPI_Wtime() - t1; @@ -324,8 +324,8 @@ void calculateSpatialTranslation( remoteTargetCellsy, remoteTargetCellsz, nPencilsX, - nPencilsY, - nPencilsZ, + nPencilsY, + nPencilsZ, dt, popID, time @@ -338,19 +338,14 @@ void calculateSpatialTranslation( // const double deltat = (MPI_Wtime() - t1) / local_propagated_cells.size(); for (size_t c=0; cparameters[CellParams::LBWEIGHTCOUNTER] += time / localCells.size(); - for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTERX] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); - mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERY] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); - mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); - } + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERX] += mpiGrid[localCells[c]]->get_number_of_all_velocity_blocks(); + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERY] += mpiGrid[localCells[c]]->get_number_of_all_velocity_blocks(); + mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += mpiGrid[localCells[c]]->get_number_of_all_velocity_blocks(); } } else { // const double deltat = MPI_Wtime() - t1; for (size_t c=0; cget_number_of_velocity_blocks(popID); - } + Real counter = mpiGrid[local_propagated_cells[c]]->get_number_of_all_velocity_blocks(); mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERX] += nPencilsX[c] * counter; mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERY] += nPencilsY[c] * counter; mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTERZ] += nPencilsZ[c] * counter; From 0ef3aa231afb8bfa0c14de2dfb11cb274b257646 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 15 Feb 2023 11:24:33 +0200 Subject: [PATCH 04/39] Provide dccrg communication weight --- grid.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grid.cpp b/grid.cpp index 33ebfb1e98..1feb83f76f 100644 --- a/grid.cpp +++ b/grid.cpp @@ -504,6 +504,7 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S //use the number of blocks. // if (P::propagateVlasovAcceleration) mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER]); + mpiGrid.set_communication_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); // else // mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); //reset counter @@ -1052,10 +1053,12 @@ void initializeStencils(dccrg::Dccrg& mpi neighborhood.push_back({{0, 0, d}}); } } + if( !mpiGrid.add_neighborhood(VLASOV_SOLVER_NEIGHBORHOOD_ID, neighborhood)){ std::cerr << "Failed to add neighborhood VLASOV_SOLVER_NEIGHBORHOOD_ID \n"; abort(); } + mpiGrid.set_partitioning_neighborhood_id(VLASOV_SOLVER_NEIGHBORHOOD_ID); // add remaining nearest neighbors for DIST_FUNC neighborhood for (int z = -1; z <= 1; z++) { From 84cc0b4badc640266d6b6702d2cccafae78754e6 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 28 Feb 2023 15:25:39 +0200 Subject: [PATCH 05/39] Scale LB weight to time everywhere --- vlasovsolver/vlasovmover.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index 7cc32dbbae..b800b7f764 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -309,25 +309,33 @@ void calculateSpatialTranslation( time ); } - + if (Parameters::prepareForRebalance == true) { + // Blocks need to be scaled to time since acceleration uses time + const double deltat {MPI_Wtime() - t1}; + double totalBlocks {0.0}; + if(P::amrMaxSpatialRefLevel == 0) { -// const double deltat = (MPI_Wtime() - t1) / local_propagated_cells.size(); for (size_t c=0; cparameters[CellParams::LBWEIGHTCOUNTER] += time / localCells.size(); for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTER] += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); + totalBlocks += mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID); + } + } + for (size_t c=0; cparameters[CellParams::LBWEIGHTCOUNTER] += deltat * mpiGrid[localCells[c]]->get_number_of_velocity_blocks(popID) / totalBlocks; } } } else { -// const double deltat = MPI_Wtime() - t1; for (size_t c=0; cget_number_of_velocity_blocks(popID); + totalBlocks += nPencils[c] * mpiGrid[local_propagated_cells[c]]->get_number_of_velocity_blocks(popID); + } + } + for (size_t c=0; cparameters[CellParams::LBWEIGHTCOUNTER] += deltat * nPencils[c] * mpiGrid[local_propagated_cells[c]]->get_number_of_velocity_blocks(popID) / totalBlocks; } - mpiGrid[local_propagated_cells[c]]->parameters[CellParams::LBWEIGHTCOUNTER] += nPencils[c] * counter; -// mpiGrid[localCells[c]]->parameters[CellParams::LBWEIGHTCOUNTER] += time / localCells.size(); } } } From 5d93ed0df490ca59615b40b31773ba4c76724780 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 3 Mar 2023 15:37:37 +0200 Subject: [PATCH 06/39] Better Vlasov neighbors --- grid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grid.cpp b/grid.cpp index 1feb83f76f..3f595e58c0 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1058,7 +1058,7 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood VLASOV_SOLVER_NEIGHBORHOOD_ID \n"; abort(); } - mpiGrid.set_partitioning_neighborhood_id(VLASOV_SOLVER_NEIGHBORHOOD_ID); + mpiGrid.set_partitioning_neighborhood_id(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); // add remaining nearest neighbors for DIST_FUNC neighborhood for (int z = -1; z <= 1; z++) { From 0ca960898a39fd9e683579b9efbca5af325ce235 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 1 Jun 2023 14:22:18 +0300 Subject: [PATCH 07/39] Parameter for HIER --- grid.cpp | 1 + parameters.cpp | 3 +++ parameters.h | 1 + 3 files changed, 5 insertions(+) diff --git a/grid.cpp b/grid.cpp index 3f595e58c0..b67b70d7f4 100644 --- a/grid.cpp +++ b/grid.cpp @@ -146,6 +146,7 @@ void initializeGrids( mpiGrid.set_initial_length(grid_length) .set_load_balancing_method(&P::loadBalanceAlgorithm[0]) + .set_load_balancing_hier_level_zero(P::loadBalanceHierLevelZero) .set_neighborhood_length(neighborhood_size) .set_maximum_refinement_level(P::amrMaxSpatialRefLevel) .set_periodic(sysBoundaries.isPeriodic(0), diff --git a/parameters.cpp b/parameters.cpp index 15ab4ea4ff..b0d873b332 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -131,6 +131,7 @@ bool P::isRestart = false; int P::writeAsFloat = false; int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); +int P::loadBalanceHierLevelZero = 0; std::map P::loadBalanceOptions; uint P::rebalanceInterval = numeric_limits::max(); @@ -370,6 +371,7 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RCB")); + RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); @@ -924,6 +926,7 @@ void Parameters::getParameters() { // Get load balance parameters RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); + RP::get("loadBalance.hierLevelZero", P::loadBalanceHierLevelZero); loadBalanceOptions["IMBALANCE_TOL"] = ""; RP::get("loadBalance.tolerance", loadBalanceOptions["IMBALANCE_TOL"]); RP::get("loadBalance.rebalanceInterval", P::rebalanceInterval); diff --git a/parameters.h b/parameters.h index 6fc3556e9b..ebf0449930 100644 --- a/parameters.h +++ b/parameters.h @@ -154,6 +154,7 @@ struct Parameters { in the Lorentz force and in the field solver.*/ static std::string loadBalanceAlgorithm; /*!< Algorithm to be used for load balance.*/ + static int loadBalanceHierLevelZero; /*!< Number of tasks for 0th level of hierarchical partition.*/ static std::map loadBalanceOptions; // Other Load balancing options static uint rebalanceInterval; /*!< Load rebalance interval (steps). */ static bool prepareForRebalance; /**< If true, propagators should measure their time consumption in preparation From d1591df8615c49bcf290a03b107fb6628585e264 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Jun 2023 13:40:25 +0300 Subject: [PATCH 08/39] Revert "Parameter for HIER" This reverts commit 4f0ccc5f8438f69ff3c42f022e3fd45d65c39cc9. --- grid.cpp | 1 - parameters.cpp | 3 --- parameters.h | 1 - 3 files changed, 5 deletions(-) diff --git a/grid.cpp b/grid.cpp index b67b70d7f4..3f595e58c0 100644 --- a/grid.cpp +++ b/grid.cpp @@ -146,7 +146,6 @@ void initializeGrids( mpiGrid.set_initial_length(grid_length) .set_load_balancing_method(&P::loadBalanceAlgorithm[0]) - .set_load_balancing_hier_level_zero(P::loadBalanceHierLevelZero) .set_neighborhood_length(neighborhood_size) .set_maximum_refinement_level(P::amrMaxSpatialRefLevel) .set_periodic(sysBoundaries.isPeriodic(0), diff --git a/parameters.cpp b/parameters.cpp index b0d873b332..15ab4ea4ff 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -131,7 +131,6 @@ bool P::isRestart = false; int P::writeAsFloat = false; int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); -int P::loadBalanceHierLevelZero = 0; std::map P::loadBalanceOptions; uint P::rebalanceInterval = numeric_limits::max(); @@ -371,7 +370,6 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RCB")); - RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); @@ -926,7 +924,6 @@ void Parameters::getParameters() { // Get load balance parameters RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); - RP::get("loadBalance.hierLevelZero", P::loadBalanceHierLevelZero); loadBalanceOptions["IMBALANCE_TOL"] = ""; RP::get("loadBalance.tolerance", loadBalanceOptions["IMBALANCE_TOL"]); RP::get("loadBalance.rebalanceInterval", P::rebalanceInterval); diff --git a/parameters.h b/parameters.h index ebf0449930..6fc3556e9b 100644 --- a/parameters.h +++ b/parameters.h @@ -154,7 +154,6 @@ struct Parameters { in the Lorentz force and in the field solver.*/ static std::string loadBalanceAlgorithm; /*!< Algorithm to be used for load balance.*/ - static int loadBalanceHierLevelZero; /*!< Number of tasks for 0th level of hierarchical partition.*/ static std::map loadBalanceOptions; // Other Load balancing options static uint rebalanceInterval; /*!< Load rebalance interval (steps). */ static bool prepareForRebalance; /**< If true, propagators should measure their time consumption in preparation From 23f30cf6b7e77e56a7f5eb3ad83bd3628660bc94 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 1 Jun 2023 15:10:25 +0300 Subject: [PATCH 09/39] Dummy commit for CI --- parameters.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/parameters.cpp b/parameters.cpp index 15ab4ea4ff..9c70204c87 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -370,6 +370,7 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RCB")); + RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); From f6ca52e247d1cf17307f63a292692af3bf494ec2 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Jun 2023 13:59:09 +0300 Subject: [PATCH 10/39] Dummy From 0ac5e73a7923c44517e67ed5389b0e416c6fa9ca Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 10:48:20 +0200 Subject: [PATCH 11/39] Hypergraph branch submodule --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index 28e5d5bbd7..b848783afc 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 28e5d5bbd7aa448d203d194ee7a0ca648141b4ff +Subproject commit b848783afc08afffb85c5078d6a42973a9c0d3f4 From a6d0aefec7d37043c23792460fc32f41cd1c64f8 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 10:58:12 +0200 Subject: [PATCH 12/39] Fix errors in merge --- grid.cpp | 2 +- vlasovsolver/vlasovmover.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index 3f595e58c0..8a28e4b9e2 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1058,7 +1058,7 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood VLASOV_SOLVER_NEIGHBORHOOD_ID \n"; abort(); } - mpiGrid.set_partitioning_neighborhood_id(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); + mpiGrid.set_partitioning_neighborhood(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); // add remaining nearest neighbors for DIST_FUNC neighborhood for (int z = -1; z <= 1; z++) { diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index b800b7f764..86bb643911 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -243,7 +243,7 @@ void calculateSpatialTranslation( phiprof::Timer semilagTimer {"semilag-trans"}; - //double t1 = MPI_Wtime(); + double t1 = MPI_Wtime(); const vector& localCells = getLocalCells(); vector remoteTargetCellsx; From 7b874e6a30dfd0d72fc3ea1ba30795388c4d9e51 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 11:01:36 +0200 Subject: [PATCH 13/39] LB algorithm for CI --- parameters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parameters.cpp b/parameters.cpp index 9c70204c87..429aa8cf17 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -369,7 +369,7 @@ bool P::addParameters() { false); // Load balancing parameters - RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RCB")); + RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("HYPERGRAPH")); RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); From 6054a54360b6a37e06628611bac84527b3e87714 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 11:59:46 +0200 Subject: [PATCH 14/39] Move stencil initialization before load balancing --- grid.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index 8a28e4b9e2..1df1eb95a5 100644 --- a/grid.cpp +++ b/grid.cpp @@ -154,10 +154,12 @@ void initializeGrids( .initialize(comm) .set_geometry(geom_params); + // Hypergraph partitioning needs stencils initialized + initializeStencils(mpiGrid); phiprof::Timer refineTimer {"Refine spatial cells"}; - // We need this first as well recalculateLocalCellsCache(); + if (!P::isRestart) { if (P::amrMaxSpatialRefLevel > 0 && project.refineSpatialCells(mpiGrid)) { mpiGrid.balance_load(); @@ -175,7 +177,6 @@ void initializeGrids( // Init velocity mesh on all cells initVelocityGridGeometry(mpiGrid); - initializeStencils(mpiGrid); for (const auto& [key, value] : P::loadBalanceOptions) { mpiGrid.set_partitioning_option(key, value); From 7ea579d7a01cfa0d75836c4a8aeb9a872985ecca Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 13:28:18 +0200 Subject: [PATCH 15/39] Change partitioning neighborhood --- grid.cpp | 2 +- submodules/dccrg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index 1df1eb95a5..a47880ddb0 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1059,7 +1059,7 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood VLASOV_SOLVER_NEIGHBORHOOD_ID \n"; abort(); } - mpiGrid.set_partitioning_neighborhood(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); + mpiGrid.set_partitioning_neighborhood(VLASOV_SOLVER_NEIGHBORHOOD_ID); // add remaining nearest neighbors for DIST_FUNC neighborhood for (int z = -1; z <= 1; z++) { diff --git a/submodules/dccrg b/submodules/dccrg index b848783afc..c8e7339f16 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit b848783afc08afffb85c5078d6a42973a9c0d3f4 +Subproject commit c8e7339f167c2acee763a386a0bb668aba780363 From 1a55ec5824b73f80b3784f89fe2498769021352b Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 9 Feb 2024 13:37:37 +0200 Subject: [PATCH 16/39] Parametrize partitioning neighborhood --- grid.cpp | 4 +++- parameters.cpp | 3 +++ parameters.h | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/grid.cpp b/grid.cpp index a47880ddb0..1c77832695 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1059,7 +1059,6 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood VLASOV_SOLVER_NEIGHBORHOOD_ID \n"; abort(); } - mpiGrid.set_partitioning_neighborhood(VLASOV_SOLVER_NEIGHBORHOOD_ID); // add remaining nearest neighbors for DIST_FUNC neighborhood for (int z = -1; z <= 1; z++) { @@ -1185,6 +1184,9 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood SHIFT_P_Z_NEIGHBORHOOD_ID \n"; abort(); } + + std::cerr << "Partitioning neighborhood " + std::to_string(P::partitioningNeighborhood) + "\n"; + mpiGrid.set_partitioning_neighborhood(P::partitioningNeighborhood); } bool validateMesh(dccrg::Dccrg& mpiGrid,const uint popID) { diff --git a/parameters.cpp b/parameters.cpp index 429aa8cf17..7d2c24cbdd 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -131,6 +131,7 @@ bool P::isRestart = false; int P::writeAsFloat = false; int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); +int P::partitioningNeighborhood {FULL_NEIGHBORHOOD_ID}; std::map P::loadBalanceOptions; uint P::rebalanceInterval = numeric_limits::max(); @@ -371,6 +372,7 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("HYPERGRAPH")); RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); + RP::add("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h", FULL_NEIGHBORHOOD_ID); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); @@ -925,6 +927,7 @@ void Parameters::getParameters() { // Get load balance parameters RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); + RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhood); loadBalanceOptions["IMBALANCE_TOL"] = ""; RP::get("loadBalance.tolerance", loadBalanceOptions["IMBALANCE_TOL"]); RP::get("loadBalance.rebalanceInterval", P::rebalanceInterval); diff --git a/parameters.h b/parameters.h index 6fc3556e9b..ce91d2d565 100644 --- a/parameters.h +++ b/parameters.h @@ -154,6 +154,7 @@ struct Parameters { in the Lorentz force and in the field solver.*/ static std::string loadBalanceAlgorithm; /*!< Algorithm to be used for load balance.*/ + static int partitioningNeighborhood; /*!< Partitioning neighborhood for (hyper)graph.*/ static std::map loadBalanceOptions; // Other Load balancing options static uint rebalanceInterval; /*!< Load rebalance interval (steps). */ static bool prepareForRebalance; /**< If true, propagators should measure their time consumption in preparation From a1c7d5d9e9ca974f08c5ad7ed251a9d714abda2e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 9 Feb 2024 13:49:00 +0200 Subject: [PATCH 17/39] Correct submodule version --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index c8e7339f16..f0b1294f2c 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit c8e7339f167c2acee763a386a0bb668aba780363 +Subproject commit f0b1294f2c6550369fd7166cd2e605c241131dbf From aeb6bb7d6664b1c3905345daa8806ba7e511af4d Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 9 Feb 2024 13:52:17 +0200 Subject: [PATCH 18/39] Remove orphaned parameter --- parameters.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/parameters.cpp b/parameters.cpp index 7d2c24cbdd..2482dbf44f 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -371,7 +371,6 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("HYPERGRAPH")); - RP::add("loadBalance.hierLevelZero", "Number of tasks for 0th level hierarchical partitioning", 4); RP::add("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h", FULL_NEIGHBORHOOD_ID); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); From 6891d6a5ff096fe849598f13417b84997e6dd00e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 11:07:39 +0200 Subject: [PATCH 19/39] Update submodule --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index f0b1294f2c..fbd74e3dd7 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit f0b1294f2c6550369fd7166cd2e605c241131dbf +Subproject commit fbd74e3dd7e32764b79528963bee7d75fe22ee5a From 9a61f2122b0f3d159cc499b4247ec08cc19cc9c4 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 12:50:23 +0200 Subject: [PATCH 20/39] Multi-neighborhood partitioning --- grid.cpp | 6 ++++-- parameters.cpp | 11 ++++++++--- parameters.h | 2 +- submodules/dccrg | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/grid.cpp b/grid.cpp index 1c77832695..16a2f159cb 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1185,8 +1185,10 @@ void initializeStencils(dccrg::Dccrg& mpi abort(); } - std::cerr << "Partitioning neighborhood " + std::to_string(P::partitioningNeighborhood) + "\n"; - mpiGrid.set_partitioning_neighborhood(P::partitioningNeighborhood); + for (auto neighborhood : P::partitioningNeighborhoods) { + std::cerr << "Partitioning neighborhood " + std::to_string(neighborhood) + "\n"; + mpiGrid.add_partitioning_neighborhood(neighborhood); + } } bool validateMesh(dccrg::Dccrg& mpiGrid,const uint popID) { diff --git a/parameters.cpp b/parameters.cpp index 2482dbf44f..4f9d4dba7a 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -131,7 +131,7 @@ bool P::isRestart = false; int P::writeAsFloat = false; int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); -int P::partitioningNeighborhood {FULL_NEIGHBORHOOD_ID}; +std::vector P::partitioningNeighborhoods {}; std::map P::loadBalanceOptions; uint P::rebalanceInterval = numeric_limits::max(); @@ -371,7 +371,7 @@ bool P::addParameters() { // Load balancing parameters RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("HYPERGRAPH")); - RP::add("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h", FULL_NEIGHBORHOOD_ID); + RP::addComposing("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h"); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); @@ -926,7 +926,12 @@ void Parameters::getParameters() { // Get load balance parameters RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); - RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhood); + RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhoods); + + if(P::partitioningNeighborhoods.empty()) { + P::partitioningNeighborhoods.push_back(FULL_NEIGHBORHOOD_ID); + } + loadBalanceOptions["IMBALANCE_TOL"] = ""; RP::get("loadBalance.tolerance", loadBalanceOptions["IMBALANCE_TOL"]); RP::get("loadBalance.rebalanceInterval", P::rebalanceInterval); diff --git a/parameters.h b/parameters.h index ce91d2d565..ea6839c774 100644 --- a/parameters.h +++ b/parameters.h @@ -154,7 +154,7 @@ struct Parameters { in the Lorentz force and in the field solver.*/ static std::string loadBalanceAlgorithm; /*!< Algorithm to be used for load balance.*/ - static int partitioningNeighborhood; /*!< Partitioning neighborhood for (hyper)graph.*/ + static std::vector partitioningNeighborhoods; /*!< Partitioning neighborhood for (hyper)graph.*/ static std::map loadBalanceOptions; // Other Load balancing options static uint rebalanceInterval; /*!< Load rebalance interval (steps). */ static bool prepareForRebalance; /**< If true, propagators should measure their time consumption in preparation diff --git a/submodules/dccrg b/submodules/dccrg index fbd74e3dd7..1c82767185 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit fbd74e3dd7e32764b79528963bee7d75fe22ee5a +Subproject commit 1c8276718538f014f2f2b0f4c85ce557a1ca6e10 From 70d0702007eb5f74aceec744f366da1687716f00 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 19:10:25 +0200 Subject: [PATCH 21/39] Attempt to fix crashes, remove debug message --- grid.cpp | 1 - submodules/dccrg | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index 16a2f159cb..e39f6b9895 100644 --- a/grid.cpp +++ b/grid.cpp @@ -1186,7 +1186,6 @@ void initializeStencils(dccrg::Dccrg& mpi } for (auto neighborhood : P::partitioningNeighborhoods) { - std::cerr << "Partitioning neighborhood " + std::to_string(neighborhood) + "\n"; mpiGrid.add_partitioning_neighborhood(neighborhood); } } diff --git a/submodules/dccrg b/submodules/dccrg index 1c82767185..6ac81458ed 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 1c8276718538f014f2f2b0f4c85ce557a1ca6e10 +Subproject commit 6ac81458ed0ac2e8f23cc37a26228e420773ec9a From 6864a87a0999e74324666908c6c9a0462e7fc4d0 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Feb 2024 00:33:45 +0200 Subject: [PATCH 22/39] Update submodule --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index 6ac81458ed..b2647b7d0f 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 6ac81458ed0ac2e8f23cc37a26228e420773ec9a +Subproject commit b2647b7d0fe33c47c39e7daf6fcda97ee53eb8a5 From 418cb14f8d5e2f98c86bfe87fcff8a2615c57e3e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Feb 2024 11:49:28 +0200 Subject: [PATCH 23/39] Use RIB by default --- parameters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parameters.cpp b/parameters.cpp index 4f9d4dba7a..568b8692cf 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -370,7 +370,7 @@ bool P::addParameters() { false); // Load balancing parameters - RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("HYPERGRAPH")); + RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RIB")); RP::addComposing("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h"); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); From 89b6197288b45483ef0115c257627abe3cb1cbf2 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 26 Feb 2024 13:14:13 +0200 Subject: [PATCH 24/39] Output reducer for system boundary communication --- common.h | 1 + datareduction/datareducer.cpp | 7 +++++++ parameters.cpp | 4 ++-- sysboundary/sysboundary.cpp | 9 +++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/common.h b/common.h index cc0fe58b77..45fe772fc7 100644 --- a/common.h +++ b/common.h @@ -191,6 +191,7 @@ namespace CellParams { AMR_TRANSLATE_COMM_X, /*! < Flag to include this cell in AMR pre-translate communication */ AMR_TRANSLATE_COMM_Y, /*! < Flag to include this cell in AMR pre-translate communication */ AMR_TRANSLATE_COMM_Z, /*! < Flag to include this cell in AMR pre-translate communication */ + SYSBOUNDARIES_COMM, // < Flag if cell is included in sysboundaries communication CONNECTION, /*!< Magnetic connection. See TracingPointConnectionType for assigned values. */ CONNECTION_FW_X, /*!< Endpoint x (forward-propagated) for the magnetic connection tracer*/ CONNECTION_FW_Y, /*!< Endpoint y (forward-propagated) for the magnetic connection tracer*/ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 7c9c8ff334..ec298e76bf 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -244,6 +244,13 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } + if(P::systemWriteAllDROs || lowercase == "vg_sysboundaries_comm") { // Flag for sysboundaries vspace communication + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_sysboundaries_comm",CellParams::SYSBOUNDARIES_COMM, 1)); + outputReducer->addMetadata(outputReducer->size()-1,"","","SysboundariesComm","1.0"); + if(!P::systemWriteAllDROs) { + continue; + } + } if(P::systemWriteAllDROs || lowercase == "fg_rhom") { // Overall mass density (summed over all populations) outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_rhom",[]( FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, diff --git a/parameters.cpp b/parameters.cpp index 568b8692cf..6bb15cc6de 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -405,7 +405,7 @@ bool P::addParameters() { "ig_precipitation ig_deltaphi "+ "ig_inplanecurrent ig_b ig_e vg_drift vg_ionospherecoupling vg_connection vg_fluxrope fg_curvature "+ "vg_amr_drho vg_amr_du vg_amr_dpsq vg_amr_dbsq vg_amr_db vg_amr_alpha vg_amr_reflevel vg_amr_jperb "+ - "vg_amr_translate_comm vg_gridcoordinates fg_gridcoordinates "); + "vg_amr_translate_comm vg_sysboundaries_comm vg_gridcoordinates fg_gridcoordinates "); RP::addComposing( "variables_deprecated.output", @@ -928,7 +928,7 @@ void Parameters::getParameters() { RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhoods); - if(P::partitioningNeighborhoods.empty()) { + if(P::partitioningNeighborhoods.empty() && (P::loadBalanceAlgorithm == "GRAPH" || P::loadBalanceAlgorithm == "HYPERGRAPH")) { P::partitioningNeighborhoods.push_back(FULL_NEIGHBORHOOD_ID); } diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index 3bd7dbf56e..7970d789e8 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -673,6 +673,15 @@ void SysBoundary::applySysBoundaryVlasovConditions( SpatialCell::set_mpi_transfer_type(Transfer::CELL_PARAMETERS | Transfer::POP_METADATA | Transfer::CELL_SYSBOUNDARYFLAG, true); mpiGrid.update_copies_of_remote_neighbors(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID); + // Mark cells that are communicating velocity blocks on system boundaries + for (auto& cell : mpiGrid.get_local_cells_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID)) { + mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = mpiGrid[cell]->sysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2; + } + + for (auto& cell : mpiGrid.get_local_cells_not_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID)) { + mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = false; + } + // Loop over existing particle species for (uint popID = 0; popID < getObjectWrapper().particleSpecies.size(); ++popID) { SpatialCell::setCommunicatedSpecies(popID); From e794fc62d613f9ab6cd13b5a3bc7e46b67b4651b Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 26 Feb 2024 13:16:14 +0200 Subject: [PATCH 25/39] Per-cell partitioning neighborhoods --- grid.cpp | 34 +++++++++++++++++++++++++++------- grid.h | 2 ++ submodules/dccrg | 2 +- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/grid.cpp b/grid.cpp index e39f6b9895..0b681a9c51 100644 --- a/grid.cpp +++ b/grid.cpp @@ -160,6 +160,8 @@ void initializeGrids( phiprof::Timer refineTimer {"Refine spatial cells"}; recalculateLocalCellsCache(); + setPartitioningNeighborhoods(mpiGrid); + if (!P::isRestart) { if (P::amrMaxSpatialRefLevel > 0 && project.refineSpatialCells(mpiGrid)) { mpiGrid.balance_load(); @@ -183,6 +185,9 @@ void initializeGrids( } phiprof::Timer initialLBTimer {"Initial load-balancing"}; if (myRank == MASTER_RANK) logFile << "(INIT): Starting initial load balance." << endl << writeVerbose; + + // TODO: do we really need two initial LB? + setPartitioningNeighborhoods(mpiGrid); mpiGrid.balance_load(); // Direct DCCRG call, recalculate cache afterwards recalculateLocalCellsCache(); @@ -485,6 +490,24 @@ void setFaceNeighborRanks( dccrg::Dccrg& } } +void setPartitioningNeighborhoods(dccrg::Dccrg& mpiGrid) { + const vector& cells = getLocalCells(); + for (auto& cell : cells){ + mpiGrid.clear_partitioning_neighborhoods(cell); + for (auto neighborhood : P::partitioningNeighborhoods) { + // TODO: for now, layer 1 cells communicate in the extended neighborhood + // If this is ever fixed, SYSBOUNDRIES_NEIGHBORHOOD_ID needs a case + if (neighborhood == SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID) { + if (mpiGrid[cell]->sysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2) { + mpiGrid.add_partitioning_neighborhood(cell, neighborhood); + } + } else { + mpiGrid.add_partitioning_neighborhood(cell, neighborhood); + } + } + } +} + void balanceLoad(dccrg::Dccrg& mpiGrid, SysBoundary& sysBoundaries){ // Invalidate cached cell lists Parameters::meshRepartitioned = true; @@ -498,14 +521,15 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S deallocTimer.stop(); //set weights based on each cells LB weight counter + setPartitioningNeighborhoods(mpiGrid); const vector& cells = getLocalCells(); - for (size_t i=0; iparameters[CellParams::LBWEIGHTCOUNTER]); - mpiGrid.set_communication_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); + mpiGrid.set_cell_weight(cell, mpiGrid[cell]->parameters[CellParams::LBWEIGHTCOUNTER]); + mpiGrid.set_communication_weight(cell, mpiGrid[cell]->get_number_of_all_velocity_blocks()); // else // mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); //reset counter @@ -1184,10 +1208,6 @@ void initializeStencils(dccrg::Dccrg& mpi std::cerr << "Failed to add neighborhood SHIFT_P_Z_NEIGHBORHOOD_ID \n"; abort(); } - - for (auto neighborhood : P::partitioningNeighborhoods) { - mpiGrid.add_partitioning_neighborhood(neighborhood); - } } bool validateMesh(dccrg::Dccrg& mpiGrid,const uint popID) { diff --git a/grid.h b/grid.h index 94f0473019..0fa9aebef7 100644 --- a/grid.h +++ b/grid.h @@ -49,6 +49,8 @@ void initializeGrids( Project& project ); +void setPartitioningNeighborhoods(dccrg::Dccrg& mpiGrid); + /*! \brief Balance load diff --git a/submodules/dccrg b/submodules/dccrg index b2647b7d0f..71bd2b864a 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit b2647b7d0fe33c47c39e7daf6fcda97ee53eb8a5 +Subproject commit 71bd2b864a3131d1bf6e816e0d0c7ca95b4b5378 From a7bb5656246a2f21f8fbb7db69b7f3fb38393e16 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 5 Apr 2024 16:12:38 +0300 Subject: [PATCH 26/39] Better multi-weight support --- grid.cpp | 28 ++++++++++++++++------------ parameters.cpp | 7 +++++++ parameters.h | 1 + vlasiator.cpp | 16 +++++++++++----- vlasovsolver/cpu_acc_semilag.cpp | 4 ++-- vlasovsolver/vlasovmover.cpp | 2 +- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/grid.cpp b/grid.cpp index d451edd5f5..fc04ade382 100644 --- a/grid.cpp +++ b/grid.cpp @@ -153,6 +153,12 @@ void initializeGrids( sysBoundaries.isPeriodic(2)) .initialize(comm) .set_geometry(geom_params); + + mpiGrid.set_load_balance_norm(P::loadBalanceNorm); + + for (const auto& [key, value] : P::loadBalanceOptions) { + mpiGrid.set_partitioning_option(key, value); + } // Hypergraph partitioning needs stencils initialized initializeStencils(mpiGrid); @@ -179,10 +185,6 @@ void initializeGrids( // Init velocity mesh on all cells initVelocityGridGeometry(mpiGrid); - - for (const auto& [key, value] : P::loadBalanceOptions) { - mpiGrid.set_partitioning_option(key, value); - } mpiGrid.set_partitioning_option("OBJ_WEIGHTS_COMPARABLE", "1"); /** RCB_MULTICRITERIA_NORM @@ -342,7 +344,9 @@ void initializeGrids( } else if (P::writeFullBGB) { // If, instead of starting a regular simulation, we are only writing out the background field, it is enough to set a dummy load balance value of 1 here. for (size_t i=0; iparameters[CellParams::LBWEIGHTCOUNTER] = 1; + for (int j = 0; j < 3; ++i) { + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX + j] = 1; + } } } @@ -539,15 +543,15 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S //use the number of blocks. std::vector lbthreeweight; if (P::propagateVlasovAcceleration) { - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERX]); - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY]); - lbthreeweight.push_back(mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ]); + lbthreeweight.push_back(mpiGrid[cell]->parameters[CellParams::LBWEIGHTCOUNTERX]); + lbthreeweight.push_back(mpiGrid[cell]->parameters[CellParams::LBWEIGHTCOUNTERY]); + lbthreeweight.push_back(mpiGrid[cell]->parameters[CellParams::LBWEIGHTCOUNTERZ]); } else { - lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); - lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); - lbthreeweight.push_back(mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks())); + lbthreeweight.push_back(mpiGrid[cell]->get_number_of_all_velocity_blocks()); + lbthreeweight.push_back(mpiGrid[cell]->get_number_of_all_velocity_blocks()); + lbthreeweight.push_back(mpiGrid[cell]->get_number_of_all_velocity_blocks()); } - mpiGrid.set_cell_weight_vector(cells[i], lbthreeweight); + mpiGrid.set_cell_weight(cell, lbthreeweight); mpiGrid.set_communication_weight(cell, mpiGrid[cell]->get_number_of_all_velocity_blocks()); } diff --git a/parameters.cpp b/parameters.cpp index 6bb15cc6de..7c347460fa 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -133,6 +133,7 @@ int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); std::vector P::partitioningNeighborhoods {}; std::map P::loadBalanceOptions; +int P::loadBalanceNorm {1}; uint P::rebalanceInterval = numeric_limits::max(); vector P::outputVariableList; @@ -373,6 +374,8 @@ bool P::addParameters() { RP::add("loadBalance.algorithm", "Load balancing algorithm to be used", string("RIB")); RP::addComposing("loadBalance.partitioning_neighborhood", "Neighborhood ID for (hyper)graph partitioning, see definitions.h"); RP::add("loadBalance.tolerance", "Load imbalance tolerance", string("1.05")); + RP::add("loadBalance.weight_dim", "Dimension of object weight", string("1")); + RP::add("loadBalance.norm", "Norm to use for cell weights in load balance, default 1-norm (sum)", 1); RP::add("loadBalance.rebalanceInterval", "Load rebalance interval (steps)", 10); RP::addComposing("loadBalance.optionKey", "Zoltan option key. Has to be matched by loadBalance.optionValue."); @@ -934,8 +937,12 @@ void Parameters::getParameters() { loadBalanceOptions["IMBALANCE_TOL"] = ""; RP::get("loadBalance.tolerance", loadBalanceOptions["IMBALANCE_TOL"]); + RP::get("loadBalance.norm", P::loadBalanceNorm); RP::get("loadBalance.rebalanceInterval", P::rebalanceInterval); + loadBalanceOptions["OBJ_WEIGHT_DIM"] = ""; + RP::get("loadBalance.weight_dim", loadBalanceOptions["OBJ_WEIGHT_DIM"]); + std::vector loadBalanceKeys; std::vector loadBalanceValues; RP::get("loadBalance.optionKey", loadBalanceKeys); diff --git a/parameters.h b/parameters.h index ea6839c774..7bce3cae6c 100644 --- a/parameters.h +++ b/parameters.h @@ -156,6 +156,7 @@ struct Parameters { static std::string loadBalanceAlgorithm; /*!< Algorithm to be used for load balance.*/ static std::vector partitioningNeighborhoods; /*!< Partitioning neighborhood for (hyper)graph.*/ static std::map loadBalanceOptions; // Other Load balancing options + static int loadBalanceNorm; // Norm to use for cell weights static uint rebalanceInterval; /*!< Load rebalance interval (steps). */ static bool prepareForRebalance; /**< If true, propagators should measure their time consumption in preparation * for mesh repartitioning.*/ diff --git a/vlasiator.cpp b/vlasiator.cpp index adc13a97ac..ab258193f8 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -1061,21 +1061,27 @@ int main(int argn,char* args[]) { logFile << "(LB) AMR rebalancing with heavier refinement weights." << endl; globalflags::bailingOut = false; // Reset this for (auto id : mpiGrid.get_local_cells_to_refine()) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] *= 8.0; + for (int i = 0; i < 3; ++i) { + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] *= 8.0; + } } balanceLoad(mpiGrid, sysBoundaryContainer); // We can /= 8.0 now as cells have potentially migrated. Go back to block-based count for now. for (auto id : mpiGrid.get_local_cells_to_refine()) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] = 0; - for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTER] += mpiGrid[id]->get_number_of_velocity_blocks(popID); + for (int i = 0; i < 3; ++i) { + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] = 0; + for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTERX + i] += mpiGrid[id]->get_number_of_velocity_blocks(popID); + } } } mpiGrid.cancel_refining(); if (!adaptRefinement(mpiGrid, technicalGrid, sysBoundaryContainer, *project)) { for (auto id : mpiGrid.get_local_cells_to_refine()) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] *= 8.0; + for (int i = 0; i < 3; ++i) { + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] *= 8.0; + } } continue; // Refinement failed and we're bailing out } else { diff --git a/vlasovsolver/cpu_acc_semilag.cpp b/vlasovsolver/cpu_acc_semilag.cpp index dcb66c250c..7ba5cdb611 100644 --- a/vlasovsolver/cpu_acc_semilag.cpp +++ b/vlasovsolver/cpu_acc_semilag.cpp @@ -85,7 +85,7 @@ void cpu_accelerate_cell(SpatialCell* spatial_cell, const uint popID, const uint map_order, const Real& dt) { - //double t1 = MPI_Wtime(); + double t1 = MPI_Wtime(); vmesh::VelocityMesh& vmesh = spatial_cell->get_velocity_mesh(popID); //vmesh::VelocityBlockContainer& blockContainer = spatial_cell->get_velocity_blocks(popID); @@ -162,6 +162,6 @@ void cpu_accelerate_cell(SpatialCell* spatial_cell, } if (Parameters::prepareForRebalance == true) { -// spatial_cell->parameters[CellParams::LBWEIGHTCOUNTER] += (MPI_Wtime() - t1); + //spatial_cell->parameters[CellParams::LBWEIGHTCOUNTER] += (MPI_Wtime() - t1); } } diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index 8b8f589d9d..c2b203c7b8 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -337,7 +337,7 @@ void calculateSpatialTranslation( } else { for (size_t c=0; cget_number_of_velocity_blocks(popID); + totalBlocks += (nPencilsX[c] + nPencilsY[c] + nPencilsZ[c]) * mpiGrid[local_propagated_cells[c]]->get_number_of_velocity_blocks(popID); } } for (size_t c=0; c Date: Tue, 16 Apr 2024 10:43:13 +0300 Subject: [PATCH 27/39] Submodule update --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index 71bd2b864a..f9ca438853 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 71bd2b864a3131d1bf6e816e0d0c7ca95b4b5378 +Subproject commit f9ca438853017a0e41aacd02f1ac51b3202348dd From 5d9a192c8f1c2c07a8d89b837da641690dcccfb2 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 16 Apr 2024 16:13:45 +0300 Subject: [PATCH 28/39] Default weight dimension 1 --- grid.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/grid.cpp b/grid.cpp index fc04ade382..3ea1491f42 100644 --- a/grid.cpp +++ b/grid.cpp @@ -189,7 +189,6 @@ void initializeGrids( mpiGrid.set_partitioning_option("OBJ_WEIGHTS_COMPARABLE", "1"); /** RCB_MULTICRITERIA_NORM Norm used in multicriteria algorithm; this determines how to balance the different weight constraints. Valid values are 1,2, and 3. Roughly, if the weights correspond to different phases, then the value 1 (1-norm) tries to minimize the total time (sum over all phases) while the value 3 (max-norm) attempts to minimize the worst imbalance in any phase. The 2-norm does something in between. Try a different value if you're not happy with the balance. **/ - mpiGrid.set_partitioning_option("OBJ_WEIGHT_DIM", "3"); phiprof::Timer initialLBTimer {"Initial load-balancing"}; if (myRank == MASTER_RANK) logFile << "(INIT): Starting initial load balance." << endl << writeVerbose; From af2e100f4605e0bbba3f6771b3da23b56b95ecb1 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 17 Apr 2024 13:18:40 +0300 Subject: [PATCH 29/39] Submodule update --- submodules/dccrg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/dccrg b/submodules/dccrg index 71bd2b864a..4e9a312410 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 71bd2b864a3131d1bf6e816e0d0c7ca95b4b5378 +Subproject commit 4e9a3124109e02099cee2cb16945eff9bfe055d3 From c3f2ab062bee821f6fb2e1b72e504d34fd23fb17 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 17 Apr 2024 14:29:05 +0300 Subject: [PATCH 30/39] Write LB weight when writing all DROs --- datareduction/datareducer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index e4a1fd86d5..6663281a81 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -3644,7 +3644,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } - if(lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { + if(P::diagnosticWriteAllDROs || lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); if(!P::diagnosticWriteAllDROs) { continue; From 1a1ceb77a4cc18f09f7587ce3995a4a3d89b2491 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 6 Jun 2024 12:28:58 +0300 Subject: [PATCH 31/39] Change LB approach after initialization --- grid.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index a68af4960c..363630fbb0 100644 --- a/grid.cpp +++ b/grid.cpp @@ -143,7 +143,7 @@ void initializeGrids( geom_params.level_0_cell_length[0] = P::dx_ini; geom_params.level_0_cell_length[1] = P::dy_ini; geom_params.level_0_cell_length[2] = P::dz_ini; - + mpiGrid.set_initial_length(grid_length) .set_load_balancing_method(&P::loadBalanceAlgorithm[0]) .set_neighborhood_length(neighborhood_size) @@ -160,6 +160,8 @@ void initializeGrids( mpiGrid.set_partitioning_option(key, value); } + mpiGrid.set_partitioning_option("LB_APPROACH", "PARTITION"); + // Hypergraph partitioning needs stencils initialized initializeStencils(mpiGrid); @@ -419,7 +421,9 @@ void initializeGrids( } P::dt = P::bailout_min_dt; } - + + mpiGrid.set_partitioning_option("LB_APPROACH", P::loadBalanceOptions.count("LB_APPROACH") ? P::loadBalanceOptions["LB_APPROACH"] : "REPARTITION"); + initialStateTimer.stop(); } From a53b551b08e9a5614234260ba7c1d1bb445a669e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 26 Nov 2024 14:55:03 +0200 Subject: [PATCH 32/39] Manual revert of multi-dimensional cell weights --- common.h | 4 +--- datareduction/datareducer.cpp | 6 +++--- grid.cpp | 10 +++------- ioread.cpp | 11 +---------- iowrite.cpp | 2 +- submodules/dccrg | 2 +- vlasiator.cpp | 20 ++++++-------------- vlasovsolver/vlasovmover.cpp | 8 +++----- 8 files changed, 19 insertions(+), 44 deletions(-) diff --git a/common.h b/common.h index 129c68ab3c..f33c71e842 100644 --- a/common.h +++ b/common.h @@ -194,9 +194,7 @@ namespace CellParams { MAXRDT, /*!< maximum timestep allowed in ordinary space for this cell, * this is the max allowed timestep over all particle species.*/ MAXFDT, /*!< maximum timestep allowed in ordinary space by fieldsolver for this cell**/ - LBWEIGHTCOUNTERX, /*!< Counter for storing compute time weights needed by the load balancing**/ - LBWEIGHTCOUNTERY, /*!< Counter for storing compute time weights needed by the load balancing**/ - LBWEIGHTCOUNTERZ, /*!< Counter for storing compute time weights needed by the load balancing**/ + LBWEIGHTCOUNTER, /*!< Counter for storing compute time weights needed by the load balancing**/ ISCELLSAVINGF, /*!< Value telling whether a cell is saving its distribution function when partial f data is written out. */ FSGRID_RANK, /*!< Rank of this cell in the FsGrid cartesian communicator */ FSGRID_BOUNDARYTYPE, /*!< Boundary type of this cell, as stored in the fsGrid */ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 3936363b73..e765007a2b 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -238,7 +238,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } } if(P::systemWriteAllDROs || lowercase == "vg_amr_translate_comm") { // Flag for AMR translation communication - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_translate_comm",CellParams::AMR_TRANSLATE_COMM, :1)); + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_translate_comm",CellParams::AMR_TRANSLATE_COMM, 1)); outputReducer->addMetadata(outputReducer->size()-1,"","","AMRtranslate","1.0"); if(!P::systemWriteAllDROs) { continue; @@ -446,7 +446,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(P::systemWriteAllDROs || lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight") { // Load balance metric for LB debugging - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTER, 1)); outputReducer->addMetadata(outputReducer->size()-1,"","","$\\mathrm{LB weight}$",""); if(!P::systemWriteAllDROs) { continue; @@ -3659,7 +3659,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } } if(P::diagnosticWriteAllDROs || lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTERX,3)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTER, 1)); if(!P::diagnosticWriteAllDROs) { continue; } diff --git a/grid.cpp b/grid.cpp index a98066a42f..a27314bfb1 100644 --- a/grid.cpp +++ b/grid.cpp @@ -151,7 +151,7 @@ void initializeGrids( initializeStencils(mpiGrid); phiprof::Timer refineTimer {"Refine spatial cells"}; - recalculateLocalCellsCache(); + recalculateLocalCellsCache(mpiGrid); setPartitioningNeighborhoods(mpiGrid); @@ -304,9 +304,7 @@ void initializeGrids( #pragma omp parallel for schedule(static) for (size_t i=0; iparameters[CellParams::LBWEIGHTCOUNTERX] = 0; - mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERY] = 0; - mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTERZ] = 0; + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER] = 0; } for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTERX + j] = 1; - } + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER] = 1.0; } } diff --git a/ioread.cpp b/ioread.cpp index 800c89bbb8..5b61b1c364 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -1314,16 +1314,7 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_dt2",CellParams::P_11_DT2,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_r",CellParams::P_11_R,3,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"pressure_v",CellParams::P_11_V,3,mpiGrid); } - if(success) { // Read 1 or 3 length LBweights - int LBlength = readVectorSize(file,string("LB_weight")); - if (LBlength==3) { - success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,3,mpiGrid); - } else { - success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERX,1,mpiGrid); - success=success && readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERY,1,mpiGrid); - success=success && readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTERZ,1,mpiGrid); - } - } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"LB_weight",CellParams::LBWEIGHTCOUNTER,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_v_dt",CellParams::MAXVDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_r_dt",CellParams::MAXRDT,1,mpiGrid); } if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"max_fields_dt",CellParams::MAXFDT,1,mpiGrid); } diff --git a/iowrite.cpp b/iowrite.cpp index 6c55fee969..d609149fd4 100644 --- a/iowrite.cpp +++ b/iowrite.cpp @@ -1656,7 +1656,7 @@ bool writeRestart( restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_dt2",CellParams::P_11_DT2,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_r",CellParams::P_11_R,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("pressure_v",CellParams::P_11_V,3)); - restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("LB_weight",CellParams::LBWEIGHTCOUNTERX,3)); + restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("LB_weight",CellParams::LBWEIGHTCOUNTER,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_v_dt",CellParams::MAXVDT,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_r_dt",CellParams::MAXRDT,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_fields_dt",CellParams::MAXFDT,1)); diff --git a/submodules/dccrg b/submodules/dccrg index 4e9a312410..5462b3b582 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 4e9a3124109e02099cee2cb16945eff9bfe055d3 +Subproject commit 5462b3b5825022eb481ae8d88ddab81f6fc04a6b diff --git a/vlasiator.cpp b/vlasiator.cpp index 3bdbe1e0b2..12dbd36cb6 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -1035,27 +1035,21 @@ int simulate(int argn,char* args[]) { logFile << "(LB) AMR rebalancing with heavier refinement weights." << endl; globalflags::bailingOut = false; // Reset this for (auto id : mpiGrid.get_local_cells_to_refine()) { - for (int i = 0; i < 3; ++i) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] *= 8.0; - } + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] *= 8.0; } balanceLoad(mpiGrid, sysBoundaryContainer, technicalGrid); // We can /= 8.0 now as cells have potentially migrated. Go back to block-based count for now. for (auto id : mpiGrid.get_local_cells_to_refine()) { - for (int i = 0; i < 3; ++i) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] = 0; - for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTERX + i] += mpiGrid[id]->get_number_of_velocity_blocks(popID); - } + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] = 0; + for (uint popID=0; popIDparameters[CellParams::LBWEIGHTCOUNTER] += mpiGrid[id]->get_number_of_velocity_blocks(popID); } } mpiGrid.cancel_refining(); if (!adaptRefinement(mpiGrid, technicalGrid, sysBoundaryContainer, *project)) { for (auto id : mpiGrid.get_local_cells_to_refine()) { - for (int i = 0; i < 3; ++i) { - mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTERX + i] *= 8.0; - } + mpiGrid[id]->parameters[CellParams::LBWEIGHTCOUNTER] *= 8.0; } continue; // Refinement failed and we're bailing out } else { @@ -1128,9 +1122,7 @@ int simulate(int argn,char* args[]) { } #pragma omp parallel for for (size_t c=0; cget_cell_parameters()[CellParams::LBWEIGHTCOUNTERX] = 0; - mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTERY] = 0; - mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTERZ] = 0; + mpiGrid[cells[c]]->get_cell_parameters()[CellParams::LBWEIGHTCOUNTER] = 0; } } diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index a876d0c134..57df8d6928 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -70,9 +70,7 @@ void calculateSpatialTranslation( const vector& remoteTargetCellsx, const vector& remoteTargetCellsy, const vector& remoteTargetCellsz, - vector& nPencilsX, - vector& nPencilsY, - vector& nPencilsZ, + vector& nPencils, creal dt, const uint popID, Real &time @@ -106,7 +104,7 @@ void calculateSpatialTranslation( if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsz, 2, dt,popID); // map along z// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsz, nPencilsZ, 2, dt,popID); // map along z// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsz, nPencils, 2, dt,popID); // map along z// } computeTimer.stop(); time += MPI_Wtime() - t1; @@ -150,7 +148,7 @@ void calculateSpatialTranslation( if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsx, 0,dt,popID); // map along x// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsx, nPencilsX, 0,dt,popID); // map along x// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsx, nPencils, 0,dt,popID); // map along x// } computeTimer.stop(); time += MPI_Wtime() - t1; From 3292e97608215282bda424e214b62a3ba3fae66e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 26 Nov 2024 17:56:29 +0200 Subject: [PATCH 33/39] Don't initialize stencils twice --- grid.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/grid.cpp b/grid.cpp index a27314bfb1..0a697c3e8b 100644 --- a/grid.cpp +++ b/grid.cpp @@ -181,8 +181,6 @@ void initializeGrids( /** RCB_MULTICRITERIA_NORM Norm used in multicriteria algorithm; this determines how to balance the different weight constraints. Valid values are 1,2, and 3. Roughly, if the weights correspond to different phases, then the value 1 (1-norm) tries to minimize the total time (sum over all phases) while the value 3 (max-norm) attempts to minimize the worst imbalance in any phase. The 2-norm does something in between. Try a different value if you're not happy with the balance. **/ - initializeStencils(mpiGrid); - for (const auto& [key, value] : P::loadBalanceOptions) { mpiGrid.set_partitioning_option(key, value); } From 4bc21f4e5736d69c4a6d3d1ad7134a9b05d9a8ca Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Tue, 20 May 2025 13:23:34 +0300 Subject: [PATCH 34/39] Clarifying comment --- sysboundary/sysboundary.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index f97e9cd349..81dae46ea8 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -694,6 +694,7 @@ void SysBoundary::applySysBoundaryVlasovConditions( mpiGrid.update_copies_of_remote_neighbors(Neighborhoods::SYSBOUNDARIES_EXTENDED); // Mark cells that are communicating velocity blocks on system boundaries + // This is not used for anything in the simulation, it's just for user analysis of sysboundary communication for (auto& cell : mpiGrid.get_local_cells_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID)) { mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = mpiGrid[cell]->sysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2; } From f85825da76bab16d499e1aef3e6661de422fb03d Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Tue, 20 May 2025 13:24:53 +0300 Subject: [PATCH 35/39] Attempted fix of issue where default partitioning neighborhood doesn't work as expected --- parameters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parameters.cpp b/parameters.cpp index 9173ec3f2e..b19604741c 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -1066,7 +1066,7 @@ void Parameters::getParameters() { RP::get("loadBalance.algorithm", P::loadBalanceAlgorithm); RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhoods); - if(P::partitioningNeighborhoods.empty() && (P::loadBalanceAlgorithm == "GRAPH" || P::loadBalanceAlgorithm == "HYPERGRAPH")) { + if (P::partitioningNeighborhoods.empty()) { P::partitioningNeighborhoods.push_back(FULL_NEIGHBORHOOD_ID); } From e06ab00b923325bf0d6d517fd8fbfba811930ddd Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Thu, 22 May 2025 16:27:11 +0300 Subject: [PATCH 36/39] Enum neighborhood ids --- grid.cpp | 2 +- parameters.cpp | 2 +- sysboundary/sysboundary.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grid.cpp b/grid.cpp index ade8e4692a..fd3a9398d6 100644 --- a/grid.cpp +++ b/grid.cpp @@ -504,7 +504,7 @@ void setPartitioningNeighborhoods(dccrg::DccrgsysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2) { mpiGrid.add_partitioning_neighborhood(cell, neighborhood); } diff --git a/parameters.cpp b/parameters.cpp index b19604741c..381a8782d4 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -1067,7 +1067,7 @@ void Parameters::getParameters() { RP::get("loadBalance.partitioning_neighborhood", P::partitioningNeighborhoods); if (P::partitioningNeighborhoods.empty()) { - P::partitioningNeighborhoods.push_back(FULL_NEIGHBORHOOD_ID); + P::partitioningNeighborhoods.push_back(Neighborhoods::FULL); } loadBalanceOptions["IMBALANCE_TOL"] = ""; diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index 81dae46ea8..fd0555888d 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -695,11 +695,11 @@ void SysBoundary::applySysBoundaryVlasovConditions( // Mark cells that are communicating velocity blocks on system boundaries // This is not used for anything in the simulation, it's just for user analysis of sysboundary communication - for (auto& cell : mpiGrid.get_local_cells_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID)) { + for (auto& cell : mpiGrid.get_local_cells_on_process_boundary(Neighborhoods::SYSBOUNDARIES_EXTENDED)) { mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = mpiGrid[cell]->sysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2; } - for (auto& cell : mpiGrid.get_local_cells_not_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID)) { + for (auto& cell : mpiGrid.get_local_cells_not_on_process_boundary(Neighborhoods::SYSBOUNDARIES_EXTENDED)) { mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = false; } From aab31e26c7e27528990d17f468736cab81baa46d Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Thu, 22 May 2025 16:41:52 +0300 Subject: [PATCH 37/39] Remove deprecated function --- grid.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/grid.cpp b/grid.cpp index fd3a9398d6..7b7b28e17d 100644 --- a/grid.cpp +++ b/grid.cpp @@ -66,7 +66,6 @@ using namespace std; extern Logger logFile; -void initVelocityGridGeometry(dccrg::Dccrg& mpiGrid); void initSpatialCellCoordinates(dccrg::Dccrg& mpiGrid); void initializeStencils(dccrg::Dccrg& mpiGrid); @@ -180,9 +179,6 @@ void initializeGrids( } refineTimer.stop(); - // Init velocity mesh on all cells - initVelocityGridGeometry(mpiGrid); - mpiGrid.set_partitioning_option("OBJ_WEIGHTS_COMPARABLE", "1"); for (const auto& [key, value] : P::loadBalanceOptions) { From c5f3fa63c528cb4a2a77523d4f748bccca4d32b3 Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Mon, 26 May 2025 10:39:57 +0300 Subject: [PATCH 38/39] Remove deprecated variables --- common.h | 1 - datareduction/datareducer.cpp | 7 ------- parameters.cpp | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/common.h b/common.h index bc0ad3e16d..18f24370d3 100644 --- a/common.h +++ b/common.h @@ -200,7 +200,6 @@ namespace CellParams { FSGRID_BOUNDARYTYPE, /*!< Boundary type of this cell, as stored in the fsGrid */ CELLID, /*! < DCCRG cell index */ REFINEMENT_LEVEL, /*! < Refinement level */ - AMR_TRANSLATE_COMM, /*! < Flag to include this cell in AMR pre-translate communication */ SYSBOUNDARIES_COMM, // < Flag if cell is included in sysboundaries communication CONNECTION, /*!< Magnetic connection. See TracingPointConnectionType for assigned values. */ CONNECTION_FW_X, /*!< Endpoint x (forward-propagated) for the magnetic connection tracer*/ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 387c001814..3591cd3699 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -237,13 +237,6 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } - if(P::systemWriteAllDROs || lowercase == "vg_amr_translate_comm") { // Flag for AMR translation communication - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_translate_comm",CellParams::AMR_TRANSLATE_COMM, 1)); - outputReducer->addMetadata(outputReducer->size()-1,"","","AMRtranslate","1.0"); - if(!P::systemWriteAllDROs) { - continue; - } - } if(P::systemWriteAllDROs || lowercase == "vg_sysboundaries_comm") { // Flag for sysboundaries vspace communication outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_sysboundaries_comm",CellParams::SYSBOUNDARIES_COMM, 1)); outputReducer->addMetadata(outputReducer->size()-1,"","","SysboundariesComm","1.0"); diff --git a/parameters.cpp b/parameters.cpp index 381a8782d4..3cdbcebfb2 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -444,7 +444,7 @@ bool P::addParameters() { "ig_inplanecurrent ig_b ig_e vg_drift vg_ionospherecoupling vg_connection vg_fluxrope fg_curvature "+ "vg_amr_drho vg_amr_du vg_amr_dpsq vg_amr_dbsq vg_amr_db vg_amr_alpha1 vg_amr_reflevel vg_amr_alpha2 "+ "vg_gridcoordinates fg_gridcoordinates vg_pressure_anisotropy vg_amr_vorticity " + - "vg_amr_translate_comm vg_sysboundaries_comm"); + "vg_sysboundaries_comm"); RP::addComposing( "variables_deprecated.output", From 9cfe3a64c4b03fa8984fcb80e9459e24328f7be5 Mon Sep 17 00:00:00 2001 From: Leo Kotipalo Date: Mon, 26 May 2025 10:52:24 +0300 Subject: [PATCH 39/39] Remove sysboundaries communication flag --- common.h | 1 - datareduction/datareducer.cpp | 7 ------- parameters.cpp | 3 +-- sysboundary/sysboundary.cpp | 10 ---------- 4 files changed, 1 insertion(+), 20 deletions(-) diff --git a/common.h b/common.h index 18f24370d3..2ed230154b 100644 --- a/common.h +++ b/common.h @@ -200,7 +200,6 @@ namespace CellParams { FSGRID_BOUNDARYTYPE, /*!< Boundary type of this cell, as stored in the fsGrid */ CELLID, /*! < DCCRG cell index */ REFINEMENT_LEVEL, /*! < Refinement level */ - SYSBOUNDARIES_COMM, // < Flag if cell is included in sysboundaries communication CONNECTION, /*!< Magnetic connection. See TracingPointConnectionType for assigned values. */ CONNECTION_FW_X, /*!< Endpoint x (forward-propagated) for the magnetic connection tracer*/ CONNECTION_FW_Y, /*!< Endpoint y (forward-propagated) for the magnetic connection tracer*/ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 3591cd3699..6e47b15166 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -237,13 +237,6 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } - if(P::systemWriteAllDROs || lowercase == "vg_sysboundaries_comm") { // Flag for sysboundaries vspace communication - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_sysboundaries_comm",CellParams::SYSBOUNDARIES_COMM, 1)); - outputReducer->addMetadata(outputReducer->size()-1,"","","SysboundariesComm","1.0"); - if(!P::systemWriteAllDROs) { - continue; - } - } if(P::systemWriteAllDROs || lowercase == "fg_rhom") { // Overall mass density (summed over all populations) outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_rhom",[]( FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, diff --git a/parameters.cpp b/parameters.cpp index 3cdbcebfb2..dbe81057ad 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -443,8 +443,7 @@ bool P::addParameters() { "ig_precipitation ig_deltaphi "+ "ig_inplanecurrent ig_b ig_e vg_drift vg_ionospherecoupling vg_connection vg_fluxrope fg_curvature "+ "vg_amr_drho vg_amr_du vg_amr_dpsq vg_amr_dbsq vg_amr_db vg_amr_alpha1 vg_amr_reflevel vg_amr_alpha2 "+ - "vg_gridcoordinates fg_gridcoordinates vg_pressure_anisotropy vg_amr_vorticity " + - "vg_sysboundaries_comm"); + "vg_gridcoordinates fg_gridcoordinates vg_pressure_anisotropy vg_amr_vorticity"); RP::addComposing( "variables_deprecated.output", diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index fd0555888d..8b505c6a3a 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -693,16 +693,6 @@ void SysBoundary::applySysBoundaryVlasovConditions( SpatialCell::set_mpi_transfer_type(Transfer::CELL_PARAMETERS | Transfer::POP_METADATA | Transfer::CELL_SYSBOUNDARYFLAG, true); mpiGrid.update_copies_of_remote_neighbors(Neighborhoods::SYSBOUNDARIES_EXTENDED); - // Mark cells that are communicating velocity blocks on system boundaries - // This is not used for anything in the simulation, it's just for user analysis of sysboundary communication - for (auto& cell : mpiGrid.get_local_cells_on_process_boundary(Neighborhoods::SYSBOUNDARIES_EXTENDED)) { - mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = mpiGrid[cell]->sysBoundaryLayer == 1 || mpiGrid[cell]->sysBoundaryLayer == 2; - } - - for (auto& cell : mpiGrid.get_local_cells_not_on_process_boundary(Neighborhoods::SYSBOUNDARIES_EXTENDED)) { - mpiGrid[cell]->parameters[CellParams::SYSBOUNDARIES_COMM] = false; - } - // Loop over existing particle species for (uint popID = 0; popID < getObjectWrapper().particleSpecies.size(); ++popID) { SpatialCell::setCommunicatedSpecies(popID);