From a1332752c6a4854e9e874c53f4a3b5a4edbf9b5e Mon Sep 17 00:00:00 2001 From: camierjs Date: Fri, 8 Jan 2021 18:00:11 -0800 Subject: [PATCH 01/22] AMR update --- laghos.cpp | 323 +++++++++++++++++++++++++++++++++++++++----- laghos_assembly.hpp | 15 +- laghos_solver.cpp | 251 ++++++++++++++++++++++++++++++++-- laghos_solver.hpp | 40 ++++-- 4 files changed, 568 insertions(+), 61 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 9c59791c..326adb25 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -81,6 +81,19 @@ static long GetMaxRssMB(); static void display_banner(std::ostream&); static void Checks(const int dim, const int ti, const double norm, int &checks); +// AMR addon +void AMRUpdate(BlockVector&, BlockVector&, Array&, + ParGridFunction&, ParGridFunction&, ParGridFunction&); +void GetZeroBCDofs(ParMesh*, ParFiniteElementSpace&, const int, + Array&, Array&); + +void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements); + +void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order = -1); + int main(int argc, char *argv[]) { // Initialize MPI. @@ -121,8 +134,13 @@ int main(int argc, char *argv[]) bool fom = false; bool gpu_aware_mpi = false; int dev = 0; + bool amr = false; + double amr_ref_threshold = 2e-4; + double amr_deref_threshold = 0.75; double blast_energy = 0.25; double blast_position[] = {0.0, 0.0, 0.0}; + const double amr_blast_size = 1e-10; + const int amr_nc_limit = 1; OptionsParser args(argc, argv); args.AddOption(&dim, "-dim", "--dimension", "Dimension of the problem."); @@ -192,14 +210,28 @@ int main(int argc, char *argv[]) args.AddOption(&gpu_aware_mpi, "-gam", "--gpu-aware-mpi", "-no-gam", "--no-gpu-aware-mpi", "Enable GPU aware MPI communications."); args.AddOption(&dev, "-dev", "--dev", "GPU device to use."); + args.AddOption(&amr, "-amr", "--enable-amr", "-no-amr", "--disable-amr", + "Experimental adaptive mesh refinement (problem 1 only)."); + args.AddOption(&amr_ref_threshold, "-rt", "--ref-threshold", + "AMR refinement threshold."); + args.AddOption(&amr_deref_threshold, "-dt", "--deref-threshold", + "AMR derefinement threshold (0 = no derefinement)."); args.Parse(); if (!args.Good()) { if (mpi.Root()) { args.PrintUsage(cout); } return 1; } + if (mpi.Root()) { args.PrintOptions(cout); } + if (amr && problem != 1) + { + if (mpi.Root()) { cout << "AMR only supported for problem 1." << endl; } + return 0; + } + + // Configure the device from the command line options Device backend; backend.Configure(device, dev); @@ -257,7 +289,22 @@ int main(int argc, char *argv[]) } // Refine the mesh in serial to increase the resolution. - for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } + if (!amr) + { + for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } + } + else + { + // Initial refinement for AMR demo on Sedov + mesh->EnsureNCMesh(); + for (int lev = 0; lev < rs_levels; lev++) + { + mesh->RefineAtVertex(Vertex(blast_position[0], blast_position[1], + blast_position[2]), + amr_blast_size); + } + } + const int mesh_NE = mesh->GetNE(); if (mpi.Root()) { @@ -403,6 +450,8 @@ int main(int argc, char *argv[]) if (myid == 0) { cout << "Zones min/max: " << ne_min << " " << ne_max << endl; } + const int amr_max_level = rs_levels + rp_levels; + // Define the parallel finite element spaces. We use: // - H1 (Gauss-Lobatto, continuous) for position and velocity. // - L2 (Bernstein, discontinuous) for specific internal energy. @@ -414,20 +463,8 @@ int main(int argc, char *argv[]) // Boundary conditions: all tests use v.n = 0 on the boundary, and we assume // that the boundaries are straight. Array ess_tdofs, ess_vdofs; - { - Array ess_bdr(pmesh->bdr_attributes.Max()), dofs_marker, dofs_list; - for (int d = 0; d < pmesh->Dimension(); d++) - { - // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, - // i.e., we must enforce v_x/y/z = 0 for the velocity components. - ess_bdr = 0; ess_bdr[d] = 1; - H1FESpace.GetEssentialTrueDofs(ess_bdr, dofs_list, d); - ess_tdofs.Append(dofs_list); - H1FESpace.GetEssentialVDofs(ess_bdr, dofs_marker, d); - FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); - ess_vdofs.Append(dofs_list); - } - } + const int bdr_attr_max = pmesh->bdr_attributes.Max(); + GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); // Define the explicit ODE solver used for time integration. ODESolver *ode_solver = NULL; @@ -504,26 +541,28 @@ int main(int argc, char *argv[]) // time evolution. ParGridFunction rho0_gf(&L2FESpace); FunctionCoefficient rho0_coeff(rho0); - L2_FECollection l2_fec(order_e, pmesh->Dimension()); - ParFiniteElementSpace l2_fes(pmesh, &l2_fec); - ParGridFunction l2_rho0_gf(&l2_fes), l2_e(&l2_fes); - l2_rho0_gf.ProjectCoefficient(rho0_coeff); - rho0_gf.ProjectGridFunction(l2_rho0_gf); - if (problem == 1) - { - // For the Sedov test, we use a delta function at the origin. - DeltaCoefficient e_coeff(blast_position[0], blast_position[1], - blast_position[2], blast_energy); - l2_e.ProjectCoefficient(e_coeff); - } - else { - FunctionCoefficient e_coeff(e0); - l2_e.ProjectCoefficient(e_coeff); + L2_FECollection l2_fec(order_e, pmesh->Dimension()); + ParFiniteElementSpace l2_fes(pmesh, &l2_fec); + ParGridFunction l2_rho0_gf(&l2_fes), l2_e(&l2_fes); + l2_rho0_gf.ProjectCoefficient(rho0_coeff); + rho0_gf.ProjectGridFunction(l2_rho0_gf); + if (problem == 1) + { + // For the Sedov test, we use a delta function at the origin. + DeltaCoefficient e_coeff(blast_position[0], blast_position[1], + blast_position[2], blast_energy); + l2_e.ProjectCoefficient(e_coeff); + } + else + { + FunctionCoefficient e_coeff(e0); + l2_e.ProjectCoefficient(e_coeff); + } + e_gf.ProjectGridFunction(l2_e); + // Sync the data location of e_gf with its base, S + e_gf.SyncAliasMemory(S); } - e_gf.ProjectGridFunction(l2_e); - // Sync the data location of e_gf with its base, S - e_gf.SyncAliasMemory(S); // Piecewise constant ideal gas coefficient over the Lagrangian mesh. The // gamma values are projected on function that's constant on the moving mesh. @@ -553,10 +592,20 @@ int main(int argc, char *argv[]) H1FESpace, L2FESpace, ess_tdofs, rho0_coeff, rho0_gf, mat_gf, source, cfl, - visc, vorticity, p_assembly, + visc, vorticity, + p_assembly, amr, cg_tol, cg_max_iter, ftz_tol, order_q); + if (amr) + { + // set a base for h0, this will be further divided in UpdateQuadratureData + // TODO: for AMR, the treatment of h0 needs more work + const double elem_size = 0.5; // coarse element size (TODO calculate) + const double h0 = elem_size / order_v; + hydro.SetH0(h0); + } + socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; int visport = 19916; @@ -611,7 +660,7 @@ int main(int argc, char *argv[]) bool last_step = false; int steps = 0; BlockVector S_old(S); - long mem=0, mmax=0, msum=0; + long mem = 0, mmax = 0, msum = 0; int checks = 0; for (int ti = 1; !last_step; ti++) @@ -750,6 +799,96 @@ int main(int argc, char *argv[]) } } + if (amr) + { + Vector &error_est = hydro.GetZoneMaxVisc(); + Vector v_max, v_min; + GetPerElementMinMax(v_gf, v_min, v_max); + bool mesh_changed = false; + + // make a list of elements to refine + Array refs; + for (int i = 0; i < pmesh->GetNE(); i++) + { + if (error_est(i) > amr_ref_threshold + && pmesh->pncmesh->GetElementDepth(i) < amr_max_level + && (v_min(i) < 1e-3 || ti < 50) // only refine the still area + ) + { + refs.Append(i); + } + } + + const int nref = pmesh->ReduceInt(refs.Size()); + if (nref) + { + pmesh->GeneralRefinement(refs, 1, amr_nc_limit); + mesh_changed = true; + if (myid == 0) + { + std::cout << "Refined " << nref << " elements." << std::endl; + } + } + else if (amr_deref_threshold) + { + hydro.ComputeDensity(rho_gf); + + Vector rho_max, rho_min; + GetPerElementMinMax(rho_gf, rho_min, rho_max); + + // simple derefinement based on zone maximum rho in post-shock region + const double rho_max_max = rho_max.Size() ? rho_max.Max() : 0.0; + double threshold, loc_threshold = amr_deref_threshold * rho_max_max; + MPI_Allreduce(&loc_threshold, &threshold, 1, MPI_DOUBLE, MPI_MAX, + pmesh->GetComm()); + + // make sure the blast point is never derefined + Array elements; + FindElementsWithVertex(pmesh, Vertex(blast_position[0], + blast_position[1], + blast_position[2]), + amr_blast_size, elements); + for (int i = 0; i < elements.Size(); i++) + { + int index = elements[i]; + if (index >= 0) { rho_max(index) = 1e10; } + } + + // also, only derefine where the mesh is in motion, i.e. after the shock + for (int i = 0; i < pmesh->GetNE(); i++) + { + if (v_min(i) < 0.1) { rho_max(i) = 1e10; } + } + + const int op = 2; // maximum value of fine elements + mesh_changed = pmesh->DerefineByError(rho_max, threshold, + amr_nc_limit, op); + if (mesh_changed && myid == 0) + { + std::cout << "Derefined, threshold = " << threshold << std::endl; + } + } + + if (mesh_changed) + { + // update state and operator + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + hydro.AMRUpdate(S, true); + + pmesh->Rebalance(); + + // update state and operator + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + hydro.AMRUpdate(S, false); + + GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); + + ode_solver->Init(hydro); + + H1FESpace.PrintPartitionStats(); + } + } + // Problems checks if (check) { @@ -832,6 +971,120 @@ int main(int argc, char *argv[]) return 0; } +void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs) +{ + Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; + for (int d = 0; d < pmesh->Dimension(); d++) + { + // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, + // i.e., we must enforce v_x/y/z = 0 for the velocity components. + ess_bdr = 0; ess_bdr[d] = 1; + H1.GetEssentialTrueDofs(ess_bdr, dofs_list, d); + ess_tdofs.Append(dofs_list); + H1.GetEssentialVDofs(ess_bdr, dofs_marker, d); + FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); + ess_vdofs.Append(dofs_list); + } +} + +void AMRUpdate(BlockVector &S, BlockVector &S_tmp, + Array &true_offset, + ParGridFunction &x_gf, + ParGridFunction &v_gf, + ParGridFunction &e_gf) +{ + ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); + ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + H1FESpace->Update(); + L2FESpace->Update(); + const int Vsize_h1 = H1FESpace->GetVSize(); + const int Vsize_l2 = L2FESpace->GetVSize(); + true_offset[0] = 0; + true_offset[1] = true_offset[0] + Vsize_h1; + true_offset[2] = true_offset[1] + Vsize_h1; + true_offset[3] = true_offset[2] + Vsize_l2; + S_tmp = S; + S.Update(true_offset); + const Operator* H1Update = H1FESpace->GetUpdateOperator(); + const Operator* L2Update = L2FESpace->GetUpdateOperator(); + H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); + H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); + L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); + x_gf.MakeRef(H1FESpace, S, true_offset[0]); + v_gf.MakeRef(H1FESpace, S, true_offset[1]); + e_gf.MakeRef(L2FESpace, S, true_offset[2]); + S_tmp.Update(true_offset); +} + +void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements) +{ + Array v; + + for (int i = 0; i < mesh->GetNE(); i++) + { + mesh->GetElementVertices(i, v); + for (int j = 0; j < v.Size(); j++) + { + double dist = 0.0; + for (int l = 0; l < mesh->SpaceDimension(); l++) + { + double d = vert(l) - mesh->GetVertex(v[j])[l]; + dist += d*d; + } + if (dist <= size*size) { elements.Append(i); break; } + } + } +} + +static void Pow(Vector &vec, double p) +{ + for (int i = 0; i < vec.Size(); i++) + { + vec(i) = std::pow(vec(i), p); + } +} + +void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order) +{ + const FiniteElementSpace *space = gf.FESpace(); + int ne = space->GetNE(); + + if (int_order < 0) { int_order = space->GetOrder(0) + 1; } + + elem_min.SetSize(ne); + elem_max.SetSize(ne); + + Vector vals, tmp; + for (int i = 0; i < ne; i++) + { + int geom = space->GetFE(i)->GetGeomType(); + const IntegrationRule &ir = IntRules.Get(geom, int_order); + + gf.GetValues(i, ir, vals); + + if (space->GetVDim() > 1) + { + Pow(vals, 2.0); + for (int vd = 1; vd < space->GetVDim(); vd++) + { + gf.GetValues(i, ir, tmp, vd+1); + Pow(tmp, 2.0); + vals += tmp; + } + Pow(vals, 0.5); + } + + elem_min(i) = vals.Min(); + elem_max(i) = vals.Max(); + } +} + double rho0(const Vector &x) { switch (problem) diff --git a/laghos_assembly.hpp b/laghos_assembly.hpp index 0699352e..cb983e42 100644 --- a/laghos_assembly.hpp +++ b/laghos_assembly.hpp @@ -55,10 +55,17 @@ struct QuadratureData // recomputed at every time step to achieve adaptive time stepping. double dt_est; - QuadratureData(int dim, int NE, int quads_per_el) - : Jac0inv(dim, dim, NE * quads_per_el), - stressJinvT(NE * quads_per_el, dim, dim), - rho0DetJ0w(NE * quads_per_el) { } + QuadratureData(int dim, int NE, int NQ) + : Jac0inv(dim, dim, NE * NQ), + stressJinvT(NE * NQ, dim, dim), + rho0DetJ0w(NE * NQ) { } + + void Resize(int dim, int NE, int NQ) + { + Jac0inv.SetSize(dim, dim, NE * NQ); + stressJinvT.SetSize(NE * NQ, dim, dim); + rho0DetJ0w.SetSize(NE * NQ); + } }; // This class is used only for visualization. It assembles (rho, phi) in each diff --git a/laghos_solver.cpp b/laghos_solver.cpp index f173a548..df42908e 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -100,12 +100,15 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, const bool visc, const bool vort, const bool p_assembly, + const bool amr, const double cgt, const int cgiter, double ftz, const int oq) : TimeDependentOperator(size), - H1(h1), L2(l2), H1c(H1.GetParMesh(), H1.FEColl(), 1), + H1(h1), + L2(l2), + H1c(H1.GetParMesh(), H1.FEColl(), 1), pmesh(H1.GetParMesh()), H1Vsize(H1.GetVSize()), H1TVSize(H1.TrueVSize()), @@ -116,16 +119,23 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, block_offsets(4), x_gf(&H1), ess_tdofs(ess_tdofs), - dim(pmesh->Dimension()), NE(pmesh->GetNE()), + dim(pmesh->Dimension()), l2dofs_cnt(L2.GetFE(0)->GetDof()), h1dofs_cnt(H1.GetFE(0)->GetDof()), - source_type(source), cfl(cfl), + source_type(source), + cfl(cfl), use_viscosity(visc), use_vorticity(vort), p_assembly(p_assembly), - cg_rel_tol(cgt), cg_max_iter(cgiter),ftz_tol(ftz), + amr(amr), + cg_rel_tol(cgt), + cg_max_iter(cgiter), + ftz_tol(ftz), gamma_gf(gamma_gf), + rho0(rho0_gf), + rho0_coeff(&rho0_gf), + x0_gf(&H1), Mv(&H1), Mv_spmat_copy(), Me(l2dofs_cnt, l2dofs_cnt, NE), Me_inv(l2dofs_cnt, l2dofs_cnt, NE), @@ -292,6 +302,187 @@ LagrangianHydroOperator::~LagrangianHydroOperator() } } +void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) +{ + H1c.Update(); + if (p_assembly) + { + rhs_c_gf.Update(); + dvc_gf.Update(); + } + + ParMesh *pmesh = H1.GetParMesh(); + + width = height = S.Size(); + NE = pmesh->GetNE(); + + l2dofs_cnt = (L2.GetFE(0)->GetDof()); + h1dofs_cnt = (H1.GetFE(0)->GetDof()); + //gamma_gf.Update(); + rho0.Update(); + rho0_coeff = * new GridFunctionCoefficient(&rho0); + x0_gf.Update(); + Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + + one.SetSize(L2.GetVSize()); + one.UseDevice(true); + one = 1.0; + + //x_gf->Update(); + + if (quick) { return; } + + // go back to initial mesh configuration temporarily + int own_nodes = 0; + GridFunction *x_gf = &x0_gf; + pmesh->SwapNodes(x_gf, own_nodes); + + H1Vsize = H1.GetVSize(); + H1TVSize = (H1.TrueVSize()); + H1GTVSize = (H1.GlobalTrueVSize()); + L2Vsize = (L2.GetVSize()); + L2TVSize = (L2.TrueVSize()); + L2GTVSize = (L2.GlobalTrueVSize()); + + block_offsets[0] = 0; + block_offsets[1] = block_offsets[0] + H1Vsize; + block_offsets[2] = block_offsets[1] + H1Vsize; + block_offsets[3] = block_offsets[2] + L2Vsize; + + if (p_assembly) + { + X.SetSize(H1c.GetTrueVSize()); + B.SetSize(H1c.GetTrueVSize()); + + one.SetSize(L2Vsize); + rhs.SetSize(H1Vsize); + e_rhs.SetSize(L2Vsize); + + delete qupdate; + qupdate = new QUpdate(dim, NE, Q1D, use_viscosity, use_vorticity, cfl, + &timer, gamma_gf, ir, H1, L2); + delete ForcePA; + ForcePA = new ForcePAOperator(qdata, H1, L2, ir); + delete VMassPA; + VMassPA = new MassPAOperator(H1c, ir, rho0_coeff); + delete EMassPA; + EMassPA = new MassPAOperator(L2, ir, rho0_coeff); + // Inside the above constructors for mass, there is reordering of the mesh + // nodes which is performed on the host. Since the mesh nodes are a + // subvector, so we need to sync with the rest of the base vector (which + // is assumed to be in the memory space used by the mfem::Device). + H1.GetParMesh()->GetNodes()->ReadWrite(); + // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, i.e., + // we must enforce v_x/y/z = 0 for the velocity components. + const int bdr_attr_max = H1.GetMesh()->bdr_attributes.Max(); + Array ess_bdr(bdr_attr_max); + for (int c = 0; c < dim; c++) + { + ess_bdr = 0; + ess_bdr[c] = 1; + H1c.GetEssentialTrueDofs(ess_bdr, c_tdofs[c]); + c_tdofs[c].Read(); + } + X.UseDevice(true); + B.UseDevice(true); + rhs.UseDevice(true); + e_rhs.UseDevice(true); + } + + if (!p_assembly) + { + // update mass matrix + // TODO: don't reassemble everything! + Mv.Update(); + Mv.Assemble(); + Mv_spmat_copy = Mv.SpMat(); + // update Me_inv + // TODO: do this better too + { + Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + MassIntegrator mi(rho0_coeff, &ir); + for (int e = 0; e < NE; e++) + { + DenseMatrixInverse inv(&Me(e)); + const FiniteElement &fe = *L2.GetFE(e); + ElementTransformation &Tr = *L2.GetElementTransformation(e); + mi.AssembleElementMatrix(fe, Tr, Me(e)); + inv.Factor(); + inv.GetInverseMatrix(Me_inv(e)); + } + } + } + + // resize quadrature data and make sure 'stressJinvT' will be recomputed + qdata.Resize(dim, NE, ir.GetNPoints()); + qdata_is_current = false; + + // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points + // TODO: remove code duplication + if (dim > 1 && p_assembly) + { + double vol = 0.0; + Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0, qdata, vol); + } + else + { + const int nqp = ir.GetNPoints(); + Vector rho_vals(nqp); + for (int i = 0; i < NE; i++) + { + rho0.GetValues(i, ir, rho_vals); + ElementTransformation *T = H1.GetElementTransformation(i); + for (int q = 0; q < nqp; q++) + { + const IntegrationPoint &ip = ir.IntPoint(q); + T->SetIntPoint(&ip); + + DenseMatrixInverse Jinv(T->Jacobian()); + Jinv.GetInverseMatrix(qdata.Jac0inv(i*nqp + q)); + + const double rho0DetJ0 = T->Weight() * rho_vals(q); + qdata.rho0DetJ0w(i*nqp + q) = rho0DetJ0 * ir.IntPoint(q).weight; + } + } + } + + if (p_assembly) + { + // Setup the preconditioner of the velocity mass operator. + // BC are handled by the VMassPA, so ess_tdofs here can be empty. + Array ess_tdofs; + VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); + CG_VMass.SetPreconditioner(*VMassPA_Jprec); + + CG_VMass.SetOperator(*VMassPA); + CG_VMass.SetRelTol(cg_rel_tol); + CG_VMass.SetAbsTol(0.0); + CG_VMass.SetMaxIter(cg_max_iter); + CG_VMass.SetPrintLevel(-1); + + CG_EMass.SetOperator(*EMassPA); + CG_EMass.iterative_mode = false; + CG_EMass.SetRelTol(cg_rel_tol); + CG_EMass.SetAbsTol(0.0); + CG_EMass.SetMaxIter(cg_max_iter); + CG_EMass.SetPrintLevel(-1); + } + else + { + ForceIntegrator *fi = new ForceIntegrator(qdata); + fi->SetIntRule(&ir); + Force.AddDomainIntegrator(fi); + // Make a dummy assembly to figure out the sparsity. + Force.Assemble(0); + Force.Finalize(0); + } + + // swap back to deformed mesh configuration + pmesh->SwapNodes(x_gf, own_nodes); +} + void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const { // Make sure that the mesh positions correspond to the ones in S. This is @@ -648,7 +839,17 @@ void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const qdata_is_current = true; forcemat_is_assembled = false; - if (dim > 1 && p_assembly) { return qupdate->UpdateQuadratureData(S, qdata); } + zone_max_visc.SetSize(NE); + zone_max_visc = 0.0; + + zone_vgrad.SetSize(NE); + zone_vgrad = 0.0; + + if (dim > 1 && p_assembly) + { + qupdate->UpdateQuadratureData(S, qdata, zone_max_visc, zone_vgrad); + return; + } // This code is only for the 1D/FA mode timer.sw_qdata.Start(); @@ -728,7 +929,7 @@ void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const p = p_b[z*nqp + q], sound_speed = cs_b[z*nqp + q]; stress = 0.0; for (int d = 0; d < dim; d++) { stress(d, d) = -p; } - double visc_coeff = 0.0; + double visc_coeff = 0.0, det_v_grad = 0.0; if (use_viscosity) { // Compression-based length scale at the point. The first @@ -746,6 +947,7 @@ void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const } sgrad_v.Symmetrize(); + det_v_grad = sgrad_v.Det(); double eig_val_data[3], eig_vec_data[9]; if (dim==1) { @@ -803,6 +1005,9 @@ void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const stressJiT(vd, gd); } } + // Track maximum artificial viscosity per zone + zone_max_visc(z_id) = std::max(visc_coeff, zone_max_visc(z_id)); + zone_vgrad(z_id) = std::max(std::abs(det_v_grad), zone_vgrad(z_id)); } ++z_id; } @@ -898,6 +1103,8 @@ void QUpdateBody(const int NE, const int e, const double* __restrict__ d_grad_v_ext, const double* __restrict__ d_Jac0inv, double *d_dt_est, + double *d_el_max_visc, + double *d_el_max_vgrad, double *d_stressJinvT) { constexpr int DIM2 = DIM*DIM; @@ -917,7 +1124,7 @@ void QUpdateBody(const int NE, const int e, const double S = sqrt(gamma * (gamma - 1.0) * E); for (int k = 0; k < DIM2; k++) { stress[k] = 0.0; } for (int d = 0; d < DIM; d++) { stress[d*DIM+d] = -P; } - double visc_coeff = 0.0; + double visc_coeff = 0.0, det_v_grad = 0.0; if (use_viscosity) { // Compression-based length scale at the point. The first @@ -936,6 +1143,7 @@ void QUpdateBody(const int NE, const int e, } kernels::Symmetrize(DIM, sgrad_v); + det_v_grad = kernels::Det(sgrad_v); if (DIM == 1) { eig_val_data[0] = sgrad_v[0]; @@ -998,6 +1206,10 @@ void QUpdateBody(const int NE, const int e, d_stressJinvT[offset] = stressJiT[vd + gd*DIM]; } } + // Track maximum artificial viscosity per zone + // !!! Race !!! + d_el_max_visc[e] = fmax(visc_coeff, d_el_max_visc[e]); + d_el_max_vgrad[e] = fmax(fabs(det_v_grad), d_el_max_vgrad[e]); } static void Rho0DetJ0Vol(const int dim, const int NE, @@ -1109,6 +1321,8 @@ void QKernel(const int NE, const int NQ, const Vector &grad_v_ext, const DenseTensor &Jac0inv, Vector &dt_est, + Vector &el_max_visc, + Vector &el_max_vgrad, DenseTensor &stressJinvT) { constexpr int DIM2 = DIM*DIM; @@ -1120,6 +1334,8 @@ void QKernel(const int NE, const int NQ, const auto d_grad_v_ext = grad_v_ext.Read(); const auto d_Jac0inv = Read(Jac0inv.GetMemory(), Jac0inv.TotalSize()); auto d_dt_est = dt_est.ReadWrite(); + auto d_max_visc = el_max_visc.ReadWrite(); + auto d_max_vgrad = el_max_vgrad.ReadWrite(); auto d_stressJinvT = Write(stressJinvT.GetMemory(), stressJinvT.TotalSize()); if (DIM == 2) { @@ -1144,7 +1360,7 @@ void QKernel(const int NE, const int NQ, compr_dir, Jpi, ph_dir, stressJiT, d_gamma, d_weights, d_Jacobians, d_rho0DetJ0w, d_e_quads, d_grad_v_ext, d_Jac0inv, - d_dt_est, d_stressJinvT); + d_dt_est, d_max_visc, d_max_vgrad, d_stressJinvT); } } MFEM_SYNC_THREAD; @@ -1175,7 +1391,7 @@ void QKernel(const int NE, const int NQ, compr_dir, Jpi, ph_dir, stressJiT, d_gamma, d_weights, d_Jacobians, d_rho0DetJ0w, d_e_quads, d_grad_v_ext, d_Jac0inv, - d_dt_est, d_stressJinvT); + d_dt_est, d_max_visc, d_max_vgrad, d_stressJinvT); } } } @@ -1184,7 +1400,10 @@ void QKernel(const int NE, const int NQ, } } -void QUpdate::UpdateQuadratureData(const Vector &S, QuadratureData &qdata) +void QUpdate::UpdateQuadratureData(const Vector &S, + QuadratureData &qdata, + Vector &zone_max_visc, + Vector &zone_max_vgrad) { timer->sw_qdata.Start(); Vector* S_p = const_cast(&S); @@ -1203,6 +1422,8 @@ void QUpdate::UpdateQuadratureData(const Vector &S, QuadratureData &qdata) q2->SetOutputLayout(QVectorLayout::byVDIM); q2->Values(e, q_e); q_dt_est = qdata.dt_est; + Vector &el_max_visc = zone_max_visc; + Vector &el_max_vgrad = zone_max_vgrad; const int id = (dim << 4) | Q1D; typedef void (*fQKernel)(const int NE, const int NQ, const bool use_viscosity, @@ -1214,7 +1435,10 @@ void QUpdate::UpdateQuadratureData(const Vector &S, QuadratureData &qdata) const Vector &Jacobians, const Vector &rho0DetJ0w, const Vector &e_quads, const Vector &grad_v_ext, const DenseTensor &Jac0inv, - Vector &dt_est, DenseTensor &stressJinvT); + Vector &dt_est, + Vector &el_max_visc, + Vector &el_max_vgrad, + DenseTensor &stressJinvT); static std::unordered_map qupdate = { {0x24,&QKernel<2,4>}, {0x26,&QKernel<2,6>}, {0x28,&QKernel<2,8>}, @@ -1228,7 +1452,10 @@ void QUpdate::UpdateQuadratureData(const Vector &S, QuadratureData &qdata) qupdate[id](NE, NQ, use_viscosity, use_vorticity, qdata.h0, h1order, cfl, infinity, gamma_gf, ir.GetWeights(), q_dx, qdata.rho0DetJ0w, q_e, q_dv, - qdata.Jac0inv, q_dt_est, qdata.stressJinvT); + qdata.Jac0inv, + q_dt_est, + el_max_visc, el_max_vgrad, + qdata.stressJinvT); qdata.dt_est = q_dt_est.Min(); timer->sw_qdata.Stop(); timer->quad_tstep += NE; diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 2bbd1c1c..3aabfc50 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -89,7 +89,9 @@ class QUpdate q2(L2.GetQuadratureInterpolator(ir)), gamma_gf(gamma_gf) { } - void UpdateQuadratureData(const Vector &S, QuadratureData &qdata); + void UpdateQuadratureData(const Vector &S, + QuadratureData &qdata, + Vector&,Vector&); }; // Given a solutions state (x, v, e), this class performs all necessary @@ -101,23 +103,31 @@ class LagrangianHydroOperator : public TimeDependentOperator mutable ParFiniteElementSpace H1c; ParMesh *pmesh; // FE spaces local and global sizes - const int H1Vsize; - const int H1TVSize; - const HYPRE_Int H1GTVSize; - const int L2Vsize; - const int L2TVSize; - const HYPRE_Int L2GTVSize; + int H1Vsize; + int H1TVSize; + HYPRE_Int H1GTVSize; + int L2Vsize; + int L2TVSize; + HYPRE_Int L2GTVSize; Array block_offsets; // Reference to the current mesh configuration. mutable ParGridFunction x_gf; const Array &ess_tdofs; - const int dim, NE, l2dofs_cnt, h1dofs_cnt, source_type; + int NE; + const int dim; + int l2dofs_cnt, h1dofs_cnt; + const int source_type; const double cfl; - const bool use_viscosity, use_vorticity, p_assembly; + const bool use_viscosity, use_vorticity, p_assembly, amr; const double cg_rel_tol; const int cg_max_iter; const double ftz_tol; const ParGridFunction &gamma_gf; + + ParGridFunction &rho0; + ParGridFunction x0_gf; // copy of initial mesh position + GridFunctionCoefficient rho0_coeff; // TODO: remove when Mv update improved + // Velocity mass matrix and local inverses of the energy mass matrices. These // are constant in time, due to the pointwise mass conservation property. mutable ParBilinearForm Mv; @@ -147,6 +157,7 @@ class LagrangianHydroOperator : public TimeDependentOperator mutable Vector X, B, one, rhs, e_rhs; mutable ParGridFunction rhs_c_gf, dvc_gf; mutable Array c_tdofs[3]; + mutable Vector zone_max_visc, zone_vgrad; virtual void ComputeMaterialProperties(int nvalues, const double gamma[], const double rho[], const double e[], @@ -172,7 +183,8 @@ class LagrangianHydroOperator : public TimeDependentOperator ParGridFunction &gamma_gf, const int source, const double cfl, - const bool visc, const bool vort, const bool pa, + const bool visc, const bool vort, + const bool p_assembly, const bool amr, const double cgt, const int cgiter, double ftz_tol, const int order_q); ~LagrangianHydroOperator(); @@ -202,6 +214,14 @@ class LagrangianHydroOperator : public TimeDependentOperator const Array &GetBlockOffsets() const { return block_offsets; } void PrintTimingData(bool IamRoot, int steps, const bool fom) const; + + void SetH0(double h0) { qdata.h0 = h0; } + double GetH0() const { return qdata.h0; } + + Vector& GetZoneMaxVisc() { return zone_max_visc; } + Vector& GetZoneVGrad() { return zone_vgrad; } + + void AMRUpdate(const Vector&, const bool quick); }; // TaylorCoefficient used in the 2D Taylor-Green problem. From 6d368e0bd30483051c73db626f7c45175f8bfc46 Mon Sep 17 00:00:00 2001 From: camierjs Date: Mon, 11 Jan 2021 16:25:50 -0800 Subject: [PATCH 02/22] AMR/PA first run --- laghos.cpp | 101 ++++++++++++++++++++++-- laghos_assembly.cpp | 20 ++++- laghos_solver.cpp | 181 +++++++++++++++++++++++++++++--------------- 3 files changed, 232 insertions(+), 70 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 326adb25..f69e881d 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -63,6 +63,7 @@ #include #include #include "laghos_solver.hpp" +#include "general/debug.hpp" using std::cout; using std::endl; @@ -141,7 +142,7 @@ int main(int argc, char *argv[]) double blast_position[] = {0.0, 0.0, 0.0}; const double amr_blast_size = 1e-10; const int amr_nc_limit = 1; - + dbg(); OptionsParser args(argc, argv); args.AddOption(&dim, "-dim", "--dimension", "Dimension of the problem."); args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use."); @@ -295,7 +296,7 @@ int main(int argc, char *argv[]) } else { - // Initial refinement for AMR demo on Sedov + dbg("Initial refinement for AMR demo on Sedov"); mesh->EnsureNCMesh(); for (int lev = 0; lev < rs_levels; lev++) { @@ -502,12 +503,14 @@ int main(int argc, char *argv[]) // - 2 -> specific internal energy const int Vsize_l2 = L2FESpace.GetVSize(); const int Vsize_h1 = H1FESpace.GetVSize(); + dbg("Init: Vsize_h1:%d Vsize_l2:%d", Vsize_h1,Vsize_l2); Array true_offset(4); true_offset[0] = 0; true_offset[1] = true_offset[0] + Vsize_h1; true_offset[2] = true_offset[1] + Vsize_h1; true_offset[3] = true_offset[2] + Vsize_l2; BlockVector S(true_offset, Device::GetMemoryType()); + dbg("Init: S:%d", S.Size()); // Define GridFunction objects for the position, velocity and specific // internal energy. There is no function for the density, as we can always @@ -589,7 +592,8 @@ int main(int argc, char *argv[]) if (impose_visc) { visc = true; } hydrodynamics::LagrangianHydroOperator hydro(S.Size(), - H1FESpace, L2FESpace, ess_tdofs, + H1FESpace, L2FESpace, + ess_tdofs, rho0_coeff, rho0_gf, mat_gf, source, cfl, visc, vorticity, @@ -599,7 +603,7 @@ int main(int argc, char *argv[]) if (amr) { - // set a base for h0, this will be further divided in UpdateQuadratureData + dbg("set a base for h0, this will be further divided in UpdateQuadratureData"); // TODO: for AMR, the treatment of h0 needs more work const double elem_size = 0.5; // coarse element size (TODO calculate) const double h0 = elem_size / order_v; @@ -663,6 +667,61 @@ int main(int argc, char *argv[]) long mem = 0, mmax = 0, msum = 0; int checks = 0; + //////////////////////////////////////////////////////////////////////// + bool fake = true; + if (fake && amr && NE==4) + { + fake = false; + dbg("\033[7mFake AMR prefix"); + // fake AMR + MFEM_VERIFY(NE == 4, "NE"); + Array refs; + refs.Append(0); // Append all 4 elements + refs.Append(1); // Append all 4 elements + refs.Append(2); // Append all 4 elements + refs.Append(3); // Append all 4 elements + + const int nref = pmesh->ReduceInt(refs.Size()); + MFEM_VERIFY(nref, "nref"); + + dbg("Refined %d elements", nref); + pmesh->GeneralRefinement(refs, 1, amr_nc_limit); + + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + dbg("mat_fes"); + mat_fes.Update(); + dbg("mat_gf update"); + mat_gf.Update(); + dbg("mat_gf ProjectCoefficient"); + mat_gf.ProjectCoefficient(mat_coeff); + + hydro.AMRUpdate(S, true); // quick + + dbg("Rebalance"); + pmesh->Rebalance(); + + dbg("Re: AMRUpdate/mat_fes/mat_gf"); + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + mat_fes.Update(); + mat_gf.Update(); + mat_gf.ProjectCoefficient(mat_coeff); + hydro.AMRUpdate(S, false); // thorough + + dbg("GetZeroBCDofs"); + GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); + + dbg("PrintPartitionStats"); + H1FESpace.PrintPartitionStats(); + + dbg("ode_solver->Init"); + ode_solver->Init(hydro); + hydro.ResetTimeStepEstimate(); + dt = hydro.GetTimeStepEstimate(S); + S_old = S; + } ////////////////////////////////////////////////////////////////////// + + + for (int ti = 1; !last_step; ti++) { if (t + dt >= t_final) @@ -872,13 +931,24 @@ int main(int argc, char *argv[]) if (mesh_changed) { // update state and operator - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + { + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + mat_fes.Update(); + mat_gf.Update(); + mat_gf.ProjectCoefficient(mat_coeff); + } + hydro.AMRUpdate(S, true); pmesh->Rebalance(); // update state and operator - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + { + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); + mat_fes.Update(); + mat_gf.Update(); + mat_gf.ProjectCoefficient(mat_coeff); + } hydro.AMRUpdate(S, false); GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); @@ -976,6 +1046,8 @@ void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, Array &ess_tdofs, Array &ess_vdofs) { + ess_tdofs.SetSize(0); + ess_vdofs.SetSize(0); Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; for (int d = 0; d < pmesh->Dimension(); d++) { @@ -998,24 +1070,41 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, { ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + dbg("\033[33mPre-update: Vsize_h1:%d Vsize_l2:%d", + H1FESpace->GetVSize(), + L2FESpace->GetVSize()); + H1FESpace->Update(); L2FESpace->Update(); + const int Vsize_h1 = H1FESpace->GetVSize(); const int Vsize_l2 = L2FESpace->GetVSize(); + dbg("\033[33mVsize_h1:%d Vsize_l2:%d", Vsize_h1, Vsize_l2); + true_offset[0] = 0; true_offset[1] = true_offset[0] + Vsize_h1; true_offset[2] = true_offset[1] + Vsize_h1; true_offset[3] = true_offset[2] + Vsize_l2; + dbg("\033[33mS_tmp:%d, S:%d", S_tmp.Size(), S.Size()); + S_tmp = S; + dbg("\033[33mS.Update"); S.Update(true_offset); + + dbg("\033[33mS_tmp:%d, S:%d", S_tmp.Size(), S.Size()); const Operator* H1Update = H1FESpace->GetUpdateOperator(); const Operator* L2Update = L2FESpace->GetUpdateOperator(); + + dbg("\033[33mH1Update->Mult"); H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); + x_gf.MakeRef(H1FESpace, S, true_offset[0]); v_gf.MakeRef(H1FESpace, S, true_offset[1]); e_gf.MakeRef(L2FESpace, S, true_offset[2]); + + dbg("\033[33mS_tmp.Update"); S_tmp.Update(true_offset); } diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index 60ee07f3..fa574312 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -17,6 +17,9 @@ #include "laghos_assembly.hpp" #include +#define MFEM_DEBUG_COLOR 226 +#include "general/debug.hpp" + namespace mfem { @@ -89,6 +92,7 @@ MassPAOperator::MassPAOperator(ParFiniteElementSpace &pfes, ess_tdofs_count(0), ess_tdofs(0) { + dbg(); pabf.SetAssemblyLevel(AssemblyLevel::PARTIAL); pabf.AddDomainIntegrator(new mfem::MassIntegrator(Q, &ir)); pabf.Assemble(); @@ -140,7 +144,11 @@ ForcePAOperator::ForcePAOperator(const QuadratureData &qdata, L2sz(L2.GetFE(0)->GetDof() * NE), L2D2Q(&L2.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR)), H1D2Q(&H1.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR)), - X(L2sz), Y(H1sz) { } + X(L2sz), Y(H1sz) +{ + dbg("NE:%d H1:%d L2:%d",NE, H1.GetVSize(), L2.GetVSize()); + dbg("H1sz:%d L2sz:%d", H1sz, L2sz); +} template static void ForceMult2D(const int NE, @@ -553,11 +561,18 @@ static void ForceMult(const int DIM, const int D1D, const int Q1D, void ForcePAOperator::Mult(const Vector &x, Vector &y) const { - if (L2R) { L2R->Mult(x, X); } + dbg("x:%d X:%d Y:%d y:%d", x.Size(), X.Size(), Y.Size(), y.Size()); + if (L2R) + { + dbg("x:%d, X:%d", x.Size(), X.Size()); + L2R->Mult(x, X); + } else { X = x; } + dbg("ForceMult"); ForceMult(dim, D1D, Q1D, L1D, D1D, NE, L2D2Q->B, H1D2Q->Bt, H1D2Q->Gt, qdata.stressJinvT, X, Y); + dbg("H1R->MultTranspose"); H1R->MultTranspose(Y, y); } @@ -958,6 +973,7 @@ static void ForceMultTranspose(const int DIM, const int D1D, const int Q1D, void ForcePAOperator::MultTranspose(const Vector &x, Vector &y) const { + dbg("x:%d, Y:%d, X:%d, y:%d", x.Size(), Y.Size(), X.Size(), y.Size()); H1R->Mult(x, Y); ForceMultTranspose(dim, D1D, Q1D, L1D, NE, L2D2Q->Bt, H1D2Q->B, H1D2Q->G, diff --git a/laghos_solver.cpp b/laghos_solver.cpp index df42908e..294c2554 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -19,6 +19,9 @@ #include "linalg/kernels.hpp" #include +#define MFEM_DEBUG_COLOR 198 +#include "general/debug.hpp" + #ifdef MFEM_USE_MPI namespace mfem @@ -134,8 +137,8 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, ftz_tol(ftz), gamma_gf(gamma_gf), rho0(rho0_gf), - rho0_coeff(&rho0_gf), x0_gf(&H1), + rho0_coeff(&rho0_gf), Mv(&H1), Mv_spmat_copy(), Me(l2dofs_cnt, l2dofs_cnt, NE), Me_inv(l2dofs_cnt, l2dofs_cnt, NE), @@ -160,6 +163,8 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, rhs_c_gf(&H1c), dvc_gf(&H1c) { + x0_gf = *pmesh->GetNodes(); + //dbg("x0_gf:"); x0_gf.Print(); block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; @@ -223,7 +228,14 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, double Volume, vol = 0.0; if (dim > 1 && p_assembly) { + dbg("dim:%d NE:%d, rho0:%d, rho0DetJ0w:%d", dim, NE, rho0.Size(), + qdata.rho0DetJ0w.Size()); + dbg("\033[31mrho0_gf:%.15e", rho0_gf*rho0_gf); + //rho0_gf.Print(); + //dbg("x_gf:"); pmesh->GetNodes()->Print(); Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); + dbg("vol:%f", vol); + //if (!amr) { MFEM_VERIFY(false,""); } } else { @@ -304,24 +316,38 @@ LagrangianHydroOperator::~LagrangianHydroOperator() void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) { - H1c.Update(); - if (p_assembly) - { - rhs_c_gf.Update(); - dvc_gf.Update(); - } + dbg("%s AMR update", quick ? "QUICK" : "THOROUGH"); - ParMesh *pmesh = H1.GetParMesh(); + ParMesh *PMESH = H1.GetParMesh(); width = height = S.Size(); - NE = pmesh->GetNE(); + NE = PMESH->GetNE(); + dbg("S:%d, NE:%d", width, NE); l2dofs_cnt = (L2.GetFE(0)->GetDof()); h1dofs_cnt = (H1.GetFE(0)->GetDof()); + + //H1.Update(); + //L2.Update(); + //gamma_gf.Update(); + ParGridFunction rho0_tmp = rho0; rho0.Update(); - rho0_coeff = * new GridFunctionCoefficient(&rho0); + const Operator *rho0_update = rho0.FESpace()->GetUpdateOperator(); + rho0_update->Mult(rho0_tmp, rho0); + + x0_gf = *PMESH->GetNodes(); + //dbg("pmesh->GetNodes():"); PMESH->GetNodes()->Print(); + //dbg("\033[7mx_gf:"); x0_gf.Print(); + /* + dbg("UPDATE"); x0_gf.Update(); + dbg("\033[7mx_gf:"); x0_gf.Print(); + ParGridFunction &x0_tmp = x0_gf; + dbg("\033[7mx_gf:"); x0_gf.Print(); + const Operator *x0_update = x0_gf.FESpace()->GetUpdateOperator(); + x0_update->Mult(x0_tmp, x0_gf);*/ + Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); @@ -329,45 +355,47 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) one.UseDevice(true); one = 1.0; - //x_gf->Update(); - - if (quick) { return; } - - // go back to initial mesh configuration temporarily - int own_nodes = 0; - GridFunction *x_gf = &x0_gf; - pmesh->SwapNodes(x_gf, own_nodes); - H1Vsize = H1.GetVSize(); H1TVSize = (H1.TrueVSize()); H1GTVSize = (H1.GlobalTrueVSize()); L2Vsize = (L2.GetVSize()); L2TVSize = (L2.TrueVSize()); L2GTVSize = (L2.GlobalTrueVSize()); + dbg("H1Vsize:%d L2Vsize:%d",H1Vsize,L2Vsize); block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; block_offsets[3] = block_offsets[2] + L2Vsize; + // resize quadrature data and make sure 'stressJinvT' will be recomputed + qdata.Resize(dim, NE, ir.GetNPoints()); + qdata_is_current = false; + if (p_assembly) { - X.SetSize(H1c.GetTrueVSize()); - B.SetSize(H1c.GetTrueVSize()); + H1c.Update(); + dvc_gf.Update(); + rhs_c_gf.Update(); + X.SetSize(H1c.GetTrueVSize()); dbg("X:%d",X.Size()); + B.SetSize(H1c.GetTrueVSize()); dbg("B:%d",B.Size()); - one.SetSize(L2Vsize); - rhs.SetSize(H1Vsize); - e_rhs.SetSize(L2Vsize); + one.SetSize(L2Vsize); dbg("one:%d", one.Size()); + rhs.SetSize(H1Vsize); dbg("rhs:%d", rhs.Size()); + e_rhs.SetSize(L2Vsize); dbg("e_rhs:%d",e_rhs.Size()); delete qupdate; qupdate = new QUpdate(dim, NE, Q1D, use_viscosity, use_vorticity, cfl, &timer, gamma_gf, ir, H1, L2); delete ForcePA; ForcePA = new ForcePAOperator(qdata, H1, L2, ir); + delete VMassPA; VMassPA = new MassPAOperator(H1c, ir, rho0_coeff); + delete EMassPA; EMassPA = new MassPAOperator(L2, ir, rho0_coeff); + // Inside the above constructors for mass, there is reordering of the mesh // nodes which is performed on the host. Since the mesh nodes are a // subvector, so we need to sync with the rest of the base vector (which @@ -390,6 +418,47 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) e_rhs.UseDevice(true); } + /////////////////////// + if (quick) { return; } + + if (p_assembly) + { + // Setup the preconditioner of the velocity mass operator. + // BC are handled by the VMassPA, so ess_tdofs here can be empty. + Array ess_tdofs; + delete VMassPA_Jprec; + VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); + CG_VMass.SetPreconditioner(*VMassPA_Jprec); + + CG_VMass.SetOperator(*VMassPA); + CG_VMass.SetRelTol(cg_rel_tol); + CG_VMass.SetAbsTol(0.0); + CG_VMass.SetMaxIter(cg_max_iter); + CG_VMass.SetPrintLevel(-1); + + CG_EMass.SetOperator(*EMassPA); + CG_EMass.iterative_mode = false; + CG_EMass.SetRelTol(cg_rel_tol); + CG_EMass.SetAbsTol(0.0); + CG_EMass.SetMaxIter(cg_max_iter); + CG_EMass.SetPrintLevel(-1); + } + else + { + ForceIntegrator *fi = new ForceIntegrator(qdata); + fi->SetIntRule(&ir); + Force.AddDomainIntegrator(fi); + // Make a dummy assembly to figure out the sparsity. + Force.Assemble(0); + Force.Finalize(0); + } + + // go back to initial mesh configuration temporarily + int own_nodes = 0; + GridFunction *x_gf = &x0_gf; + //dbg("x_gf:"); x_gf->Print(); + PMESH->SwapNodes(x_gf, own_nodes); + if (!p_assembly) { // update mass matrix @@ -415,19 +484,22 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) } } - // resize quadrature data and make sure 'stressJinvT' will be recomputed - qdata.Resize(dim, NE, ir.GetNPoints()); - qdata_is_current = false; - // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication if (dim > 1 && p_assembly) { double vol = 0.0; - Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0, qdata, vol); + dbg("dim:%d NE:%d, rho0:%d, rho0DetJ0w:%d", dim, NE, rho0.Size(), + qdata.rho0DetJ0w.Size()); + dbg("\033[31mrho0:%.15e", rho0*rho0); + //rho0.Print(); + Rho0DetJ0Vol(dim, NE, ir, PMESH, L2, rho0, qdata, vol); + dbg("vol:%f", vol); } else { + // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points + // TODO: remove code duplication const int nqp = ir.GetNPoints(); Vector rho_vals(nqp); for (int i = 0; i < NE; i++) @@ -448,43 +520,13 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) } } - if (p_assembly) - { - // Setup the preconditioner of the velocity mass operator. - // BC are handled by the VMassPA, so ess_tdofs here can be empty. - Array ess_tdofs; - VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); - CG_VMass.SetPreconditioner(*VMassPA_Jprec); - - CG_VMass.SetOperator(*VMassPA); - CG_VMass.SetRelTol(cg_rel_tol); - CG_VMass.SetAbsTol(0.0); - CG_VMass.SetMaxIter(cg_max_iter); - CG_VMass.SetPrintLevel(-1); - - CG_EMass.SetOperator(*EMassPA); - CG_EMass.iterative_mode = false; - CG_EMass.SetRelTol(cg_rel_tol); - CG_EMass.SetAbsTol(0.0); - CG_EMass.SetMaxIter(cg_max_iter); - CG_EMass.SetPrintLevel(-1); - } - else - { - ForceIntegrator *fi = new ForceIntegrator(qdata); - fi->SetIntRule(&ir); - Force.AddDomainIntegrator(fi); - // Make a dummy assembly to figure out the sparsity. - Force.Assemble(0); - Force.Finalize(0); - } - // swap back to deformed mesh configuration - pmesh->SwapNodes(x_gf, own_nodes); + PMESH->SwapNodes(x_gf, own_nodes); } void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const { + dbg("S:%d, dS_dt:%d", S.Size(), dS_dt.Size()); // Make sure that the mesh positions correspond to the ones in S. This is // needed only because some mfem time integrators don't update the solution // vector at every intermediate stage (hence they don't change the mesh). @@ -507,6 +549,7 @@ void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const void LagrangianHydroOperator::SolveVelocity(const Vector &S, Vector &dS_dt) const { + dbg("H1:%d L2:%d", H1.GetVSize(), L2.GetVSize()); UpdateQuadratureData(S); AssembleForceMatrix(); // The monolithic BlockVector stores the unknown fields as follows: @@ -527,6 +570,7 @@ void LagrangianHydroOperator::SolveVelocity(const Vector &S, if (p_assembly) { timer.sw_force.Start(); + dbg("ForcePA->Mult(one(%d), rhs(%d))", one.Size(), rhs.Size()); ForcePA->Mult(one, rhs); timer.sw_force.Stop(); rhs.Neg(); @@ -1237,6 +1281,7 @@ static void Rho0DetJ0Vol(const int dim, const int NE, Memory &Jinv_m = qdata.Jac0inv.GetMemory(); const MemoryClass mc = Device::GetMemoryClass(); const int Ji_total_size = qdata.Jac0inv.TotalSize(); + dbg("NQ:%d Ji_total_size:%d", NQ, Ji_total_size); auto invJ = Reshape(Jinv_m.Write(mc, Ji_total_size), dim, dim, NQ, NE); Vector vol(NE*NQ), one(NE*NQ); auto A = Reshape(vol.Write(), NQ, NE); @@ -1256,6 +1301,11 @@ static void Rho0DetJ0Vol(const int dim, const int NE, const double J21 = J(q,0,1,e); const double J22 = J(q,1,1,e); const double det = detJ(q,e); + if (e==0 && q == 0) + { + dbg("%.15e %.15e %.15e %.15e = %.15e", + J11, J12, J21, J22, det); + } V(q,e) = W[q] * R(q,e) * det; const double r_idetJ = 1.0 / det; invJ(0,0,q,e) = J22 * r_idetJ; @@ -1405,14 +1455,21 @@ void QUpdate::UpdateQuadratureData(const Vector &S, Vector &zone_max_visc, Vector &zone_max_vgrad) { + dbg("S:%d", S.Size()); timer->sw_qdata.Start(); Vector* S_p = const_cast(&S); const int H1_size = H1.GetVSize(); + dbg("H1_size:%d", H1_size); const double h1order = (double) H1.GetOrder(0); const double infinity = std::numeric_limits::infinity(); ParGridFunction x, v, e; x.MakeRef(&H1,*S_p, 0); - H1R->Mult(x, e_vec); + dbg("x:%d, e_vec:%d", x.Size(), e_vec.Size()); + MFEM_VERIFY(H1R, "H1R"); + const Operator *H1R_ = + H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); + H1R_->Mult(x, e_vec); + dbg("q1->Derivatives"); q1->SetOutputLayout(QVectorLayout::byVDIM); q1->Derivatives(e_vec, q_dx); v.MakeRef(&H1,*S_p, H1_size); From 66a74ee7827c2d5a1317eb1af2522e41c90e6cb1 Mon Sep 17 00:00:00 2001 From: camierjs Date: Mon, 11 Jan 2021 20:02:10 -0800 Subject: [PATCH 03/22] AMR/MPI/PA update --- laghos.cpp | 106 +++++------------- laghos_assembly.cpp | 9 +- laghos_solver.cpp | 261 +++++++++++++++++++++----------------------- laghos_solver.hpp | 7 +- 4 files changed, 163 insertions(+), 220 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index f69e881d..35c9fac0 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -232,7 +232,6 @@ int main(int argc, char *argv[]) return 0; } - // Configure the device from the command line options Device backend; backend.Configure(device, dev); @@ -601,14 +600,14 @@ int main(int argc, char *argv[]) cg_tol, cg_max_iter, ftz_tol, order_q); - if (amr) + /*if (amr) { dbg("set a base for h0, this will be further divided in UpdateQuadratureData"); // TODO: for AMR, the treatment of h0 needs more work const double elem_size = 0.5; // coarse element size (TODO calculate) const double h0 = elem_size / order_v; hydro.SetH0(h0); - } + }*/ socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; @@ -634,13 +633,14 @@ int main(int argc, char *argv[]) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); - } + }/* Wx += offx; hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, "Velocity", Wx, Wy, Ww, Wh); Wx += offx; hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", Wx, Wy, Ww, Wh); +*/ } // Save data for VisIt visualization. @@ -667,61 +667,6 @@ int main(int argc, char *argv[]) long mem = 0, mmax = 0, msum = 0; int checks = 0; - //////////////////////////////////////////////////////////////////////// - bool fake = true; - if (fake && amr && NE==4) - { - fake = false; - dbg("\033[7mFake AMR prefix"); - // fake AMR - MFEM_VERIFY(NE == 4, "NE"); - Array refs; - refs.Append(0); // Append all 4 elements - refs.Append(1); // Append all 4 elements - refs.Append(2); // Append all 4 elements - refs.Append(3); // Append all 4 elements - - const int nref = pmesh->ReduceInt(refs.Size()); - MFEM_VERIFY(nref, "nref"); - - dbg("Refined %d elements", nref); - pmesh->GeneralRefinement(refs, 1, amr_nc_limit); - - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - dbg("mat_fes"); - mat_fes.Update(); - dbg("mat_gf update"); - mat_gf.Update(); - dbg("mat_gf ProjectCoefficient"); - mat_gf.ProjectCoefficient(mat_coeff); - - hydro.AMRUpdate(S, true); // quick - - dbg("Rebalance"); - pmesh->Rebalance(); - - dbg("Re: AMRUpdate/mat_fes/mat_gf"); - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - mat_fes.Update(); - mat_gf.Update(); - mat_gf.ProjectCoefficient(mat_coeff); - hydro.AMRUpdate(S, false); // thorough - - dbg("GetZeroBCDofs"); - GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); - - dbg("PrintPartitionStats"); - H1FESpace.PrintPartitionStats(); - - dbg("ode_solver->Init"); - ode_solver->Init(hydro); - hydro.ResetTimeStepEstimate(); - dt = hydro.GetTimeStepEstimate(S); - S_old = S; - } ////////////////////////////////////////////////////////////////////// - - - for (int ti = 1; !last_step; ti++) { if (t + dt >= t_final) @@ -741,6 +686,7 @@ int main(int argc, char *argv[]) // Adaptive time step control. const double dt_est = hydro.GetTimeStepEstimate(S); + dbg("dt_est:%.8e, dt:%.8e", dt_est,dt); if (dt_est < dt) { // Repeat (solve again) with a decreased time step - decrease of the @@ -810,7 +756,7 @@ int main(int argc, char *argv[]) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); - } + }/* Wx += offx; hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, "Velocity", Wx, Wy, Ww, Wh); @@ -818,17 +764,17 @@ int main(int argc, char *argv[]) hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", Wx, Wy, Ww,Wh); - Wx += offx; + Wx += offx;*/ } - if (visit) + /*if (visit) { visit_dc.SetCycle(ti); visit_dc.SetTime(t); visit_dc.Save(); - } + }*/ - if (gfprint) + /*if (gfprint) { std::ostringstream mesh_name, rho_name, v_name, e_name; mesh_name << basename << "_" << ti << "_mesh"; @@ -855,14 +801,16 @@ int main(int argc, char *argv[]) e_ofs.precision(8); e_gf.SaveAsOne(e_ofs); e_ofs.close(); - } - } + }*/ + } // last_step if (amr) { Vector &error_est = hydro.GetZoneMaxVisc(); + Vector v_max, v_min; GetPerElementMinMax(v_gf, v_min, v_max); + bool mesh_changed = false; // make a list of elements to refine @@ -871,10 +819,11 @@ int main(int argc, char *argv[]) { if (error_est(i) > amr_ref_threshold && pmesh->pncmesh->GetElementDepth(i) < amr_max_level - && (v_min(i) < 1e-3 || ti < 50) // only refine the still area + && (v_min(i) < 1e-3 /*|| ti < 50*/) // only refine the still area ) { refs.Append(i); + dbg("Refine #%d",i); } } @@ -938,7 +887,7 @@ int main(int argc, char *argv[]) mat_gf.ProjectCoefficient(mat_coeff); } - hydro.AMRUpdate(S, true); + hydro.AMRUpdate(S, true); // quick pmesh->Rebalance(); @@ -949,9 +898,19 @@ int main(int argc, char *argv[]) mat_gf.Update(); mat_gf.ProjectCoefficient(mat_coeff); } - hydro.AMRUpdate(S, false); + + hydro.AMRUpdate(S, false); // thorough GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); + /*{ + v_gf.ProjectCoefficient(v_coeff); + for (int i = 0; i < ess_vdofs.Size(); i++) + { + v_gf(ess_vdofs[i]) = 0.0; + } + // Sync the data location of v_gf with its base, S + v_gf.SyncAliasMemory(S); + }*/ ode_solver->Init(hydro); @@ -1070,32 +1029,24 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, { ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - dbg("\033[33mPre-update: Vsize_h1:%d Vsize_l2:%d", - H1FESpace->GetVSize(), - L2FESpace->GetVSize()); H1FESpace->Update(); L2FESpace->Update(); const int Vsize_h1 = H1FESpace->GetVSize(); const int Vsize_l2 = L2FESpace->GetVSize(); - dbg("\033[33mVsize_h1:%d Vsize_l2:%d", Vsize_h1, Vsize_l2); true_offset[0] = 0; true_offset[1] = true_offset[0] + Vsize_h1; true_offset[2] = true_offset[1] + Vsize_h1; true_offset[3] = true_offset[2] + Vsize_l2; - dbg("\033[33mS_tmp:%d, S:%d", S_tmp.Size(), S.Size()); S_tmp = S; - dbg("\033[33mS.Update"); S.Update(true_offset); - dbg("\033[33mS_tmp:%d, S:%d", S_tmp.Size(), S.Size()); const Operator* H1Update = H1FESpace->GetUpdateOperator(); const Operator* L2Update = L2FESpace->GetUpdateOperator(); - dbg("\033[33mH1Update->Mult"); H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); @@ -1104,7 +1055,6 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, v_gf.MakeRef(H1FESpace, S, true_offset[1]); e_gf.MakeRef(L2FESpace, S, true_offset[2]); - dbg("\033[33mS_tmp.Update"); S_tmp.Update(true_offset); } diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index fa574312..ce1b961c 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -561,18 +561,18 @@ static void ForceMult(const int DIM, const int D1D, const int Q1D, void ForcePAOperator::Mult(const Vector &x, Vector &y) const { - dbg("x:%d X:%d Y:%d y:%d", x.Size(), X.Size(), Y.Size(), y.Size()); + //dbg("x:%d X:%d Y:%d y:%d", x.Size(), X.Size(), Y.Size(), y.Size()); if (L2R) { - dbg("x:%d, X:%d", x.Size(), X.Size()); + //dbg("x:%d, X:%d", x.Size(), X.Size()); L2R->Mult(x, X); } else { X = x; } - dbg("ForceMult"); + //dbg("ForceMult"); ForceMult(dim, D1D, Q1D, L1D, D1D, NE, L2D2Q->B, H1D2Q->Bt, H1D2Q->Gt, qdata.stressJinvT, X, Y); - dbg("H1R->MultTranspose"); + //dbg("H1R->MultTranspose"); H1R->MultTranspose(Y, y); } @@ -973,7 +973,6 @@ static void ForceMultTranspose(const int DIM, const int D1D, const int Q1D, void ForcePAOperator::MultTranspose(const Vector &x, Vector &y) const { - dbg("x:%d, Y:%d, X:%d, y:%d", x.Size(), Y.Size(), X.Size(), y.Size()); H1R->Mult(x, Y); ForceMultTranspose(dim, D1D, Q1D, L1D, NE, L2D2Q->Bt, H1D2Q->B, H1D2Q->G, diff --git a/laghos_solver.cpp b/laghos_solver.cpp index 294c2554..ea321a6f 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -136,9 +136,10 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, cg_max_iter(cgiter), ftz_tol(ftz), gamma_gf(gamma_gf), - rho0(rho0_gf), + rho0_gf(rho0_gf), x0_gf(&H1), - rho0_coeff(&rho0_gf), + rho0_coeff(rho0_coeff), + rho0_gf_coeff(&rho0_gf), Mv(&H1), Mv_spmat_copy(), Me(l2dofs_cnt, l2dofs_cnt, NE), Me_inv(l2dofs_cnt, l2dofs_cnt, NE), @@ -164,7 +165,7 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, dvc_gf(&H1c) { x0_gf = *pmesh->GetNodes(); - //dbg("x0_gf:"); x0_gf.Print(); + block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; @@ -228,14 +229,7 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, double Volume, vol = 0.0; if (dim > 1 && p_assembly) { - dbg("dim:%d NE:%d, rho0:%d, rho0DetJ0w:%d", dim, NE, rho0.Size(), - qdata.rho0DetJ0w.Size()); - dbg("\033[31mrho0_gf:%.15e", rho0_gf*rho0_gf); - //rho0_gf.Print(); - //dbg("x_gf:"); pmesh->GetNodes()->Print(); Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); - dbg("vol:%f", vol); - //if (!amr) { MFEM_VERIFY(false,""); } } else { @@ -317,85 +311,76 @@ LagrangianHydroOperator::~LagrangianHydroOperator() void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) { dbg("%s AMR update", quick ? "QUICK" : "THOROUGH"); - - ParMesh *PMESH = H1.GetParMesh(); - - width = height = S.Size(); - NE = PMESH->GetNE(); dbg("S:%d, NE:%d", width, NE); - - l2dofs_cnt = (L2.GetFE(0)->GetDof()); - h1dofs_cnt = (H1.GetFE(0)->GetDof()); - - //H1.Update(); - //L2.Update(); - - //gamma_gf.Update(); - ParGridFunction rho0_tmp = rho0; - rho0.Update(); - const Operator *rho0_update = rho0.FESpace()->GetUpdateOperator(); - rho0_update->Mult(rho0_tmp, rho0); - - x0_gf = *PMESH->GetNodes(); - //dbg("pmesh->GetNodes():"); PMESH->GetNodes()->Print(); - //dbg("\033[7mx_gf:"); x0_gf.Print(); - /* - dbg("UPDATE"); - x0_gf.Update(); - dbg("\033[7mx_gf:"); x0_gf.Print(); - ParGridFunction &x0_tmp = x0_gf; - dbg("\033[7mx_gf:"); x0_gf.Print(); - const Operator *x0_update = x0_gf.FESpace()->GetUpdateOperator(); - x0_update->Mult(x0_tmp, x0_gf);*/ - - Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - - one.SetSize(L2.GetVSize()); - one.UseDevice(true); - one = 1.0; - + width = height = S.Size(); + H1c.Update(); + pmesh = H1.GetParMesh(); H1Vsize = H1.GetVSize(); - H1TVSize = (H1.TrueVSize()); - H1GTVSize = (H1.GlobalTrueVSize()); - L2Vsize = (L2.GetVSize()); - L2TVSize = (L2.TrueVSize()); - L2GTVSize = (L2.GlobalTrueVSize()); - dbg("H1Vsize:%d L2Vsize:%d",H1Vsize,L2Vsize); - + H1TVSize = H1.TrueVSize(); + H1GTVSize = H1.GlobalTrueVSize(); + L2Vsize = L2.GetVSize(); + L2TVSize = L2.TrueVSize(); + L2GTVSize = L2.GlobalTrueVSize(); block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; block_offsets[3] = block_offsets[2] + L2Vsize; - - // resize quadrature data and make sure 'stressJinvT' will be recomputed + this->x_gf.Update(); + // ess_tdofs + NE = pmesh->GetNE(); + // dim + l2dofs_cnt = (L2.GetFE(0)->GetDof()); + h1dofs_cnt = (H1.GetFE(0)->GetDof()); + // source_type + // cfl + // use_viscosity + // use_vorticity + // p_assembly + // amr + // cg_rel_tol + // cg_max_iter + // ftz_tol + gamma_gf.Update(); + rho0_gf.Update(); + x0_gf.Update(); + //rho0_coeff = *new GridFunctionCoefficient(&rho0); + // Mv + Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + // ir + // Q1D qdata.Resize(dim, NE, ir.GetNPoints()); qdata_is_current = false; + // forcemat_is_assembled + // Force + delete ForcePA; ForcePA = nullptr; + delete VMassPA; VMassPA = nullptr; + delete EMassPA; EMassPA = nullptr; + delete VMassPA_Jprec; VMassPA_Jprec = nullptr; + // CG_VMass + // CG_EMass + // timer // to update with new sizes + delete qupdate; qupdate = nullptr; + X.SetSize(H1c.GetTrueVSize()); + B.SetSize(H1c.GetTrueVSize()); + one.SetSize(L2Vsize); one.UseDevice(true); one = 1.0; + rhs.SetSize(H1Vsize); + e_rhs.SetSize(L2Vsize); + rhs_c_gf.Update(); + dvc_gf.Update(); + // c_tdofs + // zone_max_visc, zone_vgrad + + /////////////////////// + if (quick) { return; } if (p_assembly) { - H1c.Update(); - dvc_gf.Update(); - rhs_c_gf.Update(); - X.SetSize(H1c.GetTrueVSize()); dbg("X:%d",X.Size()); - B.SetSize(H1c.GetTrueVSize()); dbg("B:%d",B.Size()); - - one.SetSize(L2Vsize); dbg("one:%d", one.Size()); - rhs.SetSize(H1Vsize); dbg("rhs:%d", rhs.Size()); - e_rhs.SetSize(L2Vsize); dbg("e_rhs:%d",e_rhs.Size()); - - delete qupdate; qupdate = new QUpdate(dim, NE, Q1D, use_viscosity, use_vorticity, cfl, &timer, gamma_gf, ir, H1, L2); - delete ForcePA; ForcePA = new ForcePAOperator(qdata, H1, L2, ir); - - delete VMassPA; VMassPA = new MassPAOperator(H1c, ir, rho0_coeff); - - delete EMassPA; EMassPA = new MassPAOperator(L2, ir, rho0_coeff); - // Inside the above constructors for mass, there is reordering of the mesh // nodes which is performed on the host. Since the mesh nodes are a // subvector, so we need to sync with the rest of the base vector (which @@ -418,46 +403,10 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) e_rhs.UseDevice(true); } - /////////////////////// - if (quick) { return; } - - if (p_assembly) - { - // Setup the preconditioner of the velocity mass operator. - // BC are handled by the VMassPA, so ess_tdofs here can be empty. - Array ess_tdofs; - delete VMassPA_Jprec; - VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); - CG_VMass.SetPreconditioner(*VMassPA_Jprec); - - CG_VMass.SetOperator(*VMassPA); - CG_VMass.SetRelTol(cg_rel_tol); - CG_VMass.SetAbsTol(0.0); - CG_VMass.SetMaxIter(cg_max_iter); - CG_VMass.SetPrintLevel(-1); - - CG_EMass.SetOperator(*EMassPA); - CG_EMass.iterative_mode = false; - CG_EMass.SetRelTol(cg_rel_tol); - CG_EMass.SetAbsTol(0.0); - CG_EMass.SetMaxIter(cg_max_iter); - CG_EMass.SetPrintLevel(-1); - } - else - { - ForceIntegrator *fi = new ForceIntegrator(qdata); - fi->SetIntRule(&ir); - Force.AddDomainIntegrator(fi); - // Make a dummy assembly to figure out the sparsity. - Force.Assemble(0); - Force.Finalize(0); - } - // go back to initial mesh configuration temporarily int own_nodes = 0; GridFunction *x_gf = &x0_gf; - //dbg("x_gf:"); x_gf->Print(); - PMESH->SwapNodes(x_gf, own_nodes); + pmesh->SwapNodes(x_gf, own_nodes); if (!p_assembly) { @@ -486,17 +435,15 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication - if (dim > 1 && p_assembly) + int Ne, ne = NE; + double Volume, vol = 0.0; + /*if (dim > 1 && p_assembly) { - double vol = 0.0; - dbg("dim:%d NE:%d, rho0:%d, rho0DetJ0w:%d", dim, NE, rho0.Size(), - qdata.rho0DetJ0w.Size()); - dbg("\033[31mrho0:%.15e", rho0*rho0); - //rho0.Print(); - Rho0DetJ0Vol(dim, NE, ir, PMESH, L2, rho0, qdata, vol); - dbg("vol:%f", vol); + Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); + dbg("vol: %.15e",vol); + MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); } - else + else*/ { // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication @@ -504,7 +451,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) Vector rho_vals(nqp); for (int i = 0; i < NE; i++) { - rho0.GetValues(i, ir, rho_vals); + rho0_gf.GetValues(i, ir, rho_vals); ElementTransformation *T = H1.GetElementTransformation(i); for (int q = 0; q < nqp; q++) { @@ -518,15 +465,60 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) qdata.rho0DetJ0w(i*nqp + q) = rho0DetJ0 * ir.IntPoint(q).weight; } } + for (int e = 0; e < NE; e++) { vol += pmesh->GetElementVolume(e); } + dbg("vol: %.15e",vol); + //MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); } + MPI_Allreduce(&vol, &Volume, 1, MPI_DOUBLE, MPI_SUM, pmesh->GetComm()); + MPI_Allreduce(&ne, &Ne, 1, MPI_INT, MPI_SUM, pmesh->GetComm()); + switch (pmesh->GetElementBaseGeometry(0)) + { + case Geometry::SEGMENT: qdata.h0 = Volume / Ne; break; + case Geometry::SQUARE: qdata.h0 = sqrt(Volume / Ne); break; + case Geometry::TRIANGLE: qdata.h0 = sqrt(2.0 * Volume / Ne); break; + case Geometry::CUBE: qdata.h0 = pow(Volume / Ne, 1./3.); break; + case Geometry::TETRAHEDRON: qdata.h0 = pow(6.0 * Volume / Ne, 1./3.); break; + default: MFEM_ABORT("Unknown zone type!"); + } + qdata.h0 /= (double) H1.GetOrder(0); // swap back to deformed mesh configuration - PMESH->SwapNodes(x_gf, own_nodes); + pmesh->SwapNodes(x_gf, own_nodes); + + if (p_assembly) + { + // Setup the preconditioner of the velocity mass operator. + // BC are handled by the VMassPA, so ess_tdofs here can be empty. + Array ess_tdofs; + VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); + CG_VMass.SetPreconditioner(*VMassPA_Jprec); + + CG_VMass.SetOperator(*VMassPA); + CG_VMass.SetRelTol(cg_rel_tol); + CG_VMass.SetAbsTol(0.0); + CG_VMass.SetMaxIter(cg_max_iter); + CG_VMass.SetPrintLevel(-1); + + CG_EMass.SetOperator(*EMassPA); + CG_EMass.iterative_mode = false; + CG_EMass.SetRelTol(cg_rel_tol); + CG_EMass.SetAbsTol(0.0); + CG_EMass.SetMaxIter(cg_max_iter); + CG_EMass.SetPrintLevel(-1); + } + else + { + ForceIntegrator *fi = new ForceIntegrator(qdata); + fi->SetIntRule(&ir); + Force.AddDomainIntegrator(fi); + // Make a dummy assembly to figure out the sparsity. + Force.Assemble(0); + Force.Finalize(0); + } } void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const { - dbg("S:%d, dS_dt:%d", S.Size(), dS_dt.Size()); // Make sure that the mesh positions correspond to the ones in S. This is // needed only because some mfem time integrators don't update the solution // vector at every intermediate stage (hence they don't change the mesh). @@ -537,6 +529,7 @@ void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const ParGridFunction v; const int VsizeH1 = H1.GetVSize(); v.MakeRef(&H1, *sptr, VsizeH1); + dbg("v:%.15e", v*v); // Set dx_dt = v (explicit). ParGridFunction dx; dx.MakeRef(&H1, dS_dt, 0); @@ -549,7 +542,6 @@ void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const void LagrangianHydroOperator::SolveVelocity(const Vector &S, Vector &dS_dt) const { - dbg("H1:%d L2:%d", H1.GetVSize(), L2.GetVSize()); UpdateQuadratureData(S); AssembleForceMatrix(); // The monolithic BlockVector stores the unknown fields as follows: @@ -570,8 +562,9 @@ void LagrangianHydroOperator::SolveVelocity(const Vector &S, if (p_assembly) { timer.sw_force.Start(); - dbg("ForcePA->Mult(one(%d), rhs(%d))", one.Size(), rhs.Size()); + dbg("one: %f", one*one); ForcePA->Mult(one, rhs); + dbg("rhs: %.15e", rhs*rhs); timer.sw_force.Stop(); rhs.Neg(); @@ -683,6 +676,7 @@ void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, // Move the memory location of the subvector 'de' to the memory // location of the base vector 'dS_dt'. de.GetMemory().SyncAlias(dS_dt.GetMemory(), de.Size()); + dbg("\033[33mde: %.15e", de*de); } else // not p_assembly { @@ -1301,11 +1295,11 @@ static void Rho0DetJ0Vol(const int dim, const int NE, const double J21 = J(q,0,1,e); const double J22 = J(q,1,1,e); const double det = detJ(q,e); - if (e==0 && q == 0) + /*if (e==0 && q == 0) { dbg("%.15e %.15e %.15e %.15e = %.15e", J11, J12, J21, J22, det); - } + }*/ V(q,e) = W[q] * R(q,e) * det; const double r_idetJ = 1.0 / det; invJ(0,0,q,e) = J22 * r_idetJ; @@ -1455,21 +1449,20 @@ void QUpdate::UpdateQuadratureData(const Vector &S, Vector &zone_max_visc, Vector &zone_max_vgrad) { - dbg("S:%d", S.Size()); + //dbg("S:%d", S.Size()); timer->sw_qdata.Start(); Vector* S_p = const_cast(&S); const int H1_size = H1.GetVSize(); - dbg("H1_size:%d", H1_size); + //dbg("H1_size:%d", H1_size); const double h1order = (double) H1.GetOrder(0); const double infinity = std::numeric_limits::infinity(); ParGridFunction x, v, e; x.MakeRef(&H1,*S_p, 0); - dbg("x:%d, e_vec:%d", x.Size(), e_vec.Size()); - MFEM_VERIFY(H1R, "H1R"); - const Operator *H1R_ = - H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); - H1R_->Mult(x, e_vec); - dbg("q1->Derivatives"); + dbg("\033[32mx:%.15e",x*x); + dbg("\033[32mv:%.15e",v*v); + dbg("\033[32me:%.15e",e*e); + //dbg("x:%d, e_vec:%d", x.Size(), e_vec.Size()); + H1R->Mult(x, e_vec); q1->SetOutputLayout(QVectorLayout::byVDIM); q1->Derivatives(e_vec, q_dx); v.MakeRef(&H1,*S_p, H1_size); diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 3aabfc50..4b00600f 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -122,11 +122,12 @@ class LagrangianHydroOperator : public TimeDependentOperator const double cg_rel_tol; const int cg_max_iter; const double ftz_tol; - const ParGridFunction &gamma_gf; + ParGridFunction &gamma_gf; - ParGridFunction &rho0; + ParGridFunction rho0_gf; ParGridFunction x0_gf; // copy of initial mesh position - GridFunctionCoefficient rho0_coeff; // TODO: remove when Mv update improved + Coefficient &rho0_coeff; + GridFunctionCoefficient rho0_gf_coeff; // TODO: remove when Mv update improved // Velocity mass matrix and local inverses of the energy mass matrices. These // are constant in time, due to the pointwise mass conservation property. From 51b74f9b4a6309c8c7e426a99be316217e2b8181 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 08:38:34 -0800 Subject: [PATCH 04/22] Re-enable Rho0DetJ0Vol & DeleteGeometricFactors --- laghos_solver.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/laghos_solver.cpp b/laghos_solver.cpp index ea321a6f..ace7c899 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -315,6 +315,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) width = height = S.Size(); H1c.Update(); pmesh = H1.GetParMesh(); + pmesh->DeleteGeometricFactors(); H1Vsize = H1.GetVSize(); H1TVSize = H1.TrueVSize(); H1GTVSize = H1.GlobalTrueVSize(); @@ -437,13 +438,13 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) // TODO: remove code duplication int Ne, ne = NE; double Volume, vol = 0.0; - /*if (dim > 1 && p_assembly) + if (dim > 1 && p_assembly) { Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); dbg("vol: %.15e",vol); MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); } - else*/ + else { // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication @@ -467,7 +468,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) } for (int e = 0; e < NE; e++) { vol += pmesh->GetElementVolume(e); } dbg("vol: %.15e",vol); - //MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); + MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); } MPI_Allreduce(&vol, &Volume, 1, MPI_DOUBLE, MPI_SUM, pmesh->GetComm()); MPI_Allreduce(&ne, &Ne, 1, MPI_INT, MPI_SUM, pmesh->GetComm()); From 18be93742db150008ad92061f09047738fc20f2f Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 10:55:58 -0800 Subject: [PATCH 05/22] Parallel check volume fix --- .gitignore | 17 +++++++++++++++++ laghos_solver.cpp | 4 +--- 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..353f2f30 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*.o +*~ + +# OS-specific: Mac +*.dSYM +.DS_Store + +Laghos + +# QtCreator files +*.cflags +*.config +*.creator +*.creator.user +*.cxxflags +*.files +*.includes \ No newline at end of file diff --git a/laghos_solver.cpp b/laghos_solver.cpp index ace7c899..d4e23c2d 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -442,7 +442,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) { Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); dbg("vol: %.15e",vol); - MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); + // MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); // only in serial } else { @@ -467,8 +467,6 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) } } for (int e = 0; e < NE; e++) { vol += pmesh->GetElementVolume(e); } - dbg("vol: %.15e",vol); - MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); } MPI_Allreduce(&vol, &Volume, 1, MPI_DOUBLE, MPI_SUM, pmesh->GetComm()); MPI_Allreduce(&ne, &Ne, 1, MPI_INT, MPI_SUM, pmesh->GetComm()); From d91d8f512710030118acd1f91b7d2ae30fbf9d40 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 14:10:34 -0800 Subject: [PATCH 06/22] PA/FA AMR --- laghos.cpp | 11 ++-- laghos_assembly.cpp | 16 +----- laghos_solver.cpp | 121 ++++++++++++++++++++++---------------------- 3 files changed, 70 insertions(+), 78 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 35c9fac0..7c4edd77 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -628,7 +628,7 @@ int main(int argc, char *argv[]) vis_e.precision(8); int Wx = 0, Wy = 0; // window position const int Ww = 350, Wh = 350; // window size - int offx = Ww+10; // window offsets + //int offx = Ww+10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, @@ -751,7 +751,7 @@ int main(int argc, char *argv[]) { int Wx = 0, Wy = 0; // window position int Ww = 350, Wh = 350; // window size - int offx = Ww+10; // window offsets + //int offx = Ww+10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, @@ -887,6 +887,7 @@ int main(int argc, char *argv[]) mat_gf.ProjectCoefficient(mat_coeff); } + hydro.UpdateMesh(S); hydro.AMRUpdate(S, true); // quick pmesh->Rebalance(); @@ -899,6 +900,7 @@ int main(int argc, char *argv[]) mat_gf.ProjectCoefficient(mat_coeff); } + hydro.UpdateMesh(S); hydro.AMRUpdate(S, false); // thorough GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); @@ -1043,10 +1045,8 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, S_tmp = S; S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); @@ -1054,6 +1054,9 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, x_gf.MakeRef(H1FESpace, S, true_offset[0]); v_gf.MakeRef(H1FESpace, S, true_offset[1]); e_gf.MakeRef(L2FESpace, S, true_offset[2]); + x_gf.SyncAliasMemory(S); + v_gf.SyncAliasMemory(S); + e_gf.SyncAliasMemory(S); S_tmp.Update(true_offset); } diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index ce1b961c..6dec3ab6 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -92,7 +92,6 @@ MassPAOperator::MassPAOperator(ParFiniteElementSpace &pfes, ess_tdofs_count(0), ess_tdofs(0) { - dbg(); pabf.SetAssemblyLevel(AssemblyLevel::PARTIAL); pabf.AddDomainIntegrator(new mfem::MassIntegrator(Q, &ir)); pabf.Assemble(); @@ -144,11 +143,7 @@ ForcePAOperator::ForcePAOperator(const QuadratureData &qdata, L2sz(L2.GetFE(0)->GetDof() * NE), L2D2Q(&L2.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR)), H1D2Q(&H1.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR)), - X(L2sz), Y(H1sz) -{ - dbg("NE:%d H1:%d L2:%d",NE, H1.GetVSize(), L2.GetVSize()); - dbg("H1sz:%d L2sz:%d", H1sz, L2sz); -} + X(L2sz), Y(H1sz) { } template static void ForceMult2D(const int NE, @@ -561,18 +556,11 @@ static void ForceMult(const int DIM, const int D1D, const int Q1D, void ForcePAOperator::Mult(const Vector &x, Vector &y) const { - //dbg("x:%d X:%d Y:%d y:%d", x.Size(), X.Size(), Y.Size(), y.Size()); - if (L2R) - { - //dbg("x:%d, X:%d", x.Size(), X.Size()); - L2R->Mult(x, X); - } + if (L2R) { L2R->Mult(x, X); } else { X = x; } - //dbg("ForceMult"); ForceMult(dim, D1D, Q1D, L1D, D1D, NE, L2D2Q->B, H1D2Q->Bt, H1D2Q->Gt, qdata.stressJinvT, X, Y); - //dbg("H1R->MultTranspose"); H1R->MultTranspose(Y, y); } diff --git a/laghos_solver.cpp b/laghos_solver.cpp index d4e23c2d..99a6bbac 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -140,7 +140,8 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, x0_gf(&H1), rho0_coeff(rho0_coeff), rho0_gf_coeff(&rho0_gf), - Mv(&H1), Mv_spmat_copy(), + Mv(&H1), + Mv_spmat_copy(), Me(l2dofs_cnt, l2dofs_cnt, NE), Me_inv(l2dofs_cnt, l2dofs_cnt, NE), ir(IntRules.Get(pmesh->GetElementBaseGeometry(0), @@ -150,7 +151,9 @@ LagrangianHydroOperator::LagrangianHydroOperator(const int size, qdata_is_current(false), forcemat_is_assembled(false), Force(&L2, &H1), - ForcePA(nullptr), VMassPA(nullptr), EMassPA(nullptr), + ForcePA(nullptr), + VMassPA(nullptr), + EMassPA(nullptr), VMassPA_Jprec(nullptr), CG_VMass(H1.GetParMesh()->GetComm()), CG_EMass(L2.GetParMesh()->GetComm()), @@ -313,7 +316,6 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) dbg("%s AMR update", quick ? "QUICK" : "THOROUGH"); dbg("S:%d, NE:%d", width, NE); width = height = S.Size(); - H1c.Update(); pmesh = H1.GetParMesh(); pmesh->DeleteGeometricFactors(); H1Vsize = H1.GetVSize(); @@ -322,6 +324,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) L2Vsize = L2.GetVSize(); L2TVSize = L2.TrueVSize(); L2GTVSize = L2.GlobalTrueVSize(); + block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; @@ -344,16 +347,15 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) gamma_gf.Update(); rho0_gf.Update(); x0_gf.Update(); - //rho0_coeff = *new GridFunctionCoefficient(&rho0); - // Mv - Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + // Mv_spmat_copy + //Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + //Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); // ir // Q1D qdata.Resize(dim, NE, ir.GetNPoints()); qdata_is_current = false; - // forcemat_is_assembled - // Force + forcemat_is_assembled = false; + Force.Update(); delete ForcePA; ForcePA = nullptr; delete VMassPA; VMassPA = nullptr; delete EMassPA; EMassPA = nullptr; @@ -364,17 +366,28 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) delete qupdate; qupdate = nullptr; X.SetSize(H1c.GetTrueVSize()); B.SetSize(H1c.GetTrueVSize()); - one.SetSize(L2Vsize); one.UseDevice(true); one = 1.0; + + one.SetSize(L2Vsize); + one.UseDevice(true); + one = 1.0; + rhs.SetSize(H1Vsize); e_rhs.SetSize(L2Vsize); - rhs_c_gf.Update(); - dvc_gf.Update(); + + H1c.Update(); + //rhs_c_gf.Update(); + //dvc_gf.Update(); // c_tdofs // zone_max_visc, zone_vgrad /////////////////////// if (quick) { return; } + // go back to initial mesh configuration temporarily + int own_nodes = 0; + GridFunction *x_gf = &x0_gf; + pmesh->SwapNodes(x_gf, own_nodes); + if (p_assembly) { qupdate = new QUpdate(dim, NE, Q1D, use_viscosity, use_vorticity, cfl, @@ -403,37 +416,34 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) rhs.UseDevice(true); e_rhs.UseDevice(true); } - - // go back to initial mesh configuration temporarily - int own_nodes = 0; - GridFunction *x_gf = &x0_gf; - pmesh->SwapNodes(x_gf, own_nodes); - - if (!p_assembly) + else { // update mass matrix + // TODO: do this better too // TODO: don't reassemble everything! Mv.Update(); Mv.Assemble(); Mv_spmat_copy = Mv.SpMat(); - // update Me_inv - // TODO: do this better too + Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); + MassIntegrator mi(rho0_coeff, &ir); + for (int e = 0; e < NE; e++) { - Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - MassIntegrator mi(rho0_coeff, &ir); - for (int e = 0; e < NE; e++) - { - DenseMatrixInverse inv(&Me(e)); - const FiniteElement &fe = *L2.GetFE(e); - ElementTransformation &Tr = *L2.GetElementTransformation(e); - mi.AssembleElementMatrix(fe, Tr, Me(e)); - inv.Factor(); - inv.GetInverseMatrix(Me_inv(e)); - } + DenseMatrixInverse inv(&Me(e)); + const FiniteElement &fe = *L2.GetFE(e); + ElementTransformation &Tr = *L2.GetElementTransformation(e); + mi.AssembleElementMatrix(fe, Tr, Me(e)); + inv.Factor(); + inv.GetInverseMatrix(Me_inv(e)); } + // Standard assembly for the velocity mass matrix. + //VectorMassIntegrator *vmi = new VectorMassIntegrator(rho0_coeff, &ir); + //Mv.AddDomainIntegrator(vmi); + //Mv.Assemble(); + //Mv_spmat_copy = Mv.SpMat(); } + // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication int Ne, ne = NE; @@ -441,29 +451,23 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) if (dim > 1 && p_assembly) { Rho0DetJ0Vol(dim, NE, ir, pmesh, L2, rho0_gf, qdata, vol); - dbg("vol: %.15e",vol); - // MFEM_VERIFY(fabs(vol - 1.0)<1.e-14,""); // only in serial } else { - // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points - // TODO: remove code duplication - const int nqp = ir.GetNPoints(); - Vector rho_vals(nqp); - for (int i = 0; i < NE; i++) + const int NQ = ir.GetNPoints(); + Vector rho_vals(NQ); + for (int e = 0; e < NE; e++) { - rho0_gf.GetValues(i, ir, rho_vals); - ElementTransformation *T = H1.GetElementTransformation(i); - for (int q = 0; q < nqp; q++) + rho0_gf.GetValues(e, ir, rho_vals); + ElementTransformation *T = H1.GetElementTransformation(e); + for (int q = 0; q < NQ; q++) { const IntegrationPoint &ip = ir.IntPoint(q); T->SetIntPoint(&ip); - DenseMatrixInverse Jinv(T->Jacobian()); - Jinv.GetInverseMatrix(qdata.Jac0inv(i*nqp + q)); - + Jinv.GetInverseMatrix(qdata.Jac0inv(e*NQ + q)); const double rho0DetJ0 = T->Weight() * rho_vals(q); - qdata.rho0DetJ0w(i*nqp + q) = rho0DetJ0 * ir.IntPoint(q).weight; + qdata.rho0DetJ0w(e*NQ + q) = rho0DetJ0 * ir.IntPoint(q).weight; } } for (int e = 0; e < NE; e++) { vol += pmesh->GetElementVolume(e); } @@ -480,6 +484,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) default: MFEM_ABORT("Unknown zone type!"); } qdata.h0 /= (double) H1.GetOrder(0); + dbg("h0:%.15e", qdata.h0); // swap back to deformed mesh configuration pmesh->SwapNodes(x_gf, own_nodes); @@ -507,9 +512,9 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) } else { - ForceIntegrator *fi = new ForceIntegrator(qdata); - fi->SetIntRule(&ir); - Force.AddDomainIntegrator(fi); + //ForceIntegrator *fi = new ForceIntegrator(qdata); + //fi->SetIntRule(&ir); + //Force.AddDomainIntegrator(fi); // Make a dummy assembly to figure out the sparsity. Force.Assemble(0); Force.Finalize(0); @@ -518,6 +523,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const { + dbg("S:%.15e", S*S); // Make sure that the mesh positions correspond to the ones in S. This is // needed only because some mfem time integrators don't update the solution // vector at every intermediate stage (hence they don't change the mesh). @@ -533,8 +539,11 @@ void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const ParGridFunction dx; dx.MakeRef(&H1, dS_dt, 0); dx = v; + dbg("dx:%.15e", dx*dx); SolveVelocity(S, dS_dt); + dbg("dS_dt:%.15e", dS_dt*dS_dt); SolveEnergy(S, v, dS_dt); + dbg("dS_dt:%.15e", dS_dt*dS_dt); qdata_is_current = false; } @@ -561,9 +570,7 @@ void LagrangianHydroOperator::SolveVelocity(const Vector &S, if (p_assembly) { timer.sw_force.Start(); - dbg("one: %f", one*one); ForcePA->Mult(one, rhs); - dbg("rhs: %.15e", rhs*rhs); timer.sw_force.Stop(); rhs.Neg(); @@ -635,6 +642,7 @@ void LagrangianHydroOperator::SolveVelocity(const Vector &S, timer.H1iter += cg.GetNumIterations(); Mv.RecoverFEMSolution(X, rhs, dv); } + dbg("S:%.15e dv:%.15e", S*S, dv*dv); } void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, @@ -675,7 +683,6 @@ void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, // Move the memory location of the subvector 'de' to the memory // location of the base vector 'dS_dt'. de.GetMemory().SyncAlias(dS_dt.GetMemory(), de.Size()); - dbg("\033[33mde: %.15e", de*de); } else // not p_assembly { @@ -695,6 +702,7 @@ void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, de.SetSubVector(l2dofs, loc_de); } } + dbg("v:%.15e de:%.15e", v*v, de*de); delete e_source; } @@ -1274,7 +1282,6 @@ static void Rho0DetJ0Vol(const int dim, const int NE, Memory &Jinv_m = qdata.Jac0inv.GetMemory(); const MemoryClass mc = Device::GetMemoryClass(); const int Ji_total_size = qdata.Jac0inv.TotalSize(); - dbg("NQ:%d Ji_total_size:%d", NQ, Ji_total_size); auto invJ = Reshape(Jinv_m.Write(mc, Ji_total_size), dim, dim, NQ, NE); Vector vol(NE*NQ), one(NE*NQ); auto A = Reshape(vol.Write(), NQ, NE); @@ -1448,19 +1455,13 @@ void QUpdate::UpdateQuadratureData(const Vector &S, Vector &zone_max_visc, Vector &zone_max_vgrad) { - //dbg("S:%d", S.Size()); timer->sw_qdata.Start(); Vector* S_p = const_cast(&S); const int H1_size = H1.GetVSize(); - //dbg("H1_size:%d", H1_size); const double h1order = (double) H1.GetOrder(0); const double infinity = std::numeric_limits::infinity(); ParGridFunction x, v, e; x.MakeRef(&H1,*S_p, 0); - dbg("\033[32mx:%.15e",x*x); - dbg("\033[32mv:%.15e",v*v); - dbg("\033[32me:%.15e",e*e); - //dbg("x:%d, e_vec:%d", x.Size(), e_vec.Size()); H1R->Mult(x, e_vec); q1->SetOutputLayout(QVectorLayout::byVDIM); q1->Derivatives(e_vec, q_dx); From b870b27c295924378bcde97b11b87266fd329970 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 15:43:14 -0800 Subject: [PATCH 07/22] Cleanup --- laghos.cpp | 121 +++++++++++++++++-------------------------- laghos_assembly.cpp | 3 -- laghos_solver.cpp | 122 ++++++++++---------------------------------- 3 files changed, 74 insertions(+), 172 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 7c4edd77..a4d62bb5 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -63,7 +63,6 @@ #include #include #include "laghos_solver.hpp" -#include "general/debug.hpp" using std::cout; using std::endl; @@ -82,18 +81,19 @@ static long GetMaxRssMB(); static void display_banner(std::ostream&); static void Checks(const int dim, const int ti, const double norm, int &checks); -// AMR addon -void AMRUpdate(BlockVector&, BlockVector&, Array&, - ParGridFunction&, ParGridFunction&, ParGridFunction&); -void GetZeroBCDofs(ParMesh*, ParFiniteElementSpace&, const int, - Array&, Array&); +static void AMRUpdate(BlockVector&, BlockVector&, Array&, + ParGridFunction&, ParGridFunction&, + ParGridFunction&, ParGridFunction&); -void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements); +static void GetZeroBCDofs(ParMesh*, ParFiniteElementSpace&, const int, + Array&, Array&); -void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order = -1); +static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements); + +static void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order = -1); int main(int argc, char *argv[]) { @@ -135,14 +135,14 @@ int main(int argc, char *argv[]) bool fom = false; bool gpu_aware_mpi = false; int dev = 0; + double blast_energy = 0.25; + double blast_position[] = {0.0, 0.0, 0.0}; bool amr = false; double amr_ref_threshold = 2e-4; double amr_deref_threshold = 0.75; - double blast_energy = 0.25; - double blast_position[] = {0.0, 0.0, 0.0}; const double amr_blast_size = 1e-10; const int amr_nc_limit = 1; - dbg(); + OptionsParser args(argc, argv); args.AddOption(&dim, "-dim", "--dimension", "Dimension of the problem."); args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use."); @@ -211,11 +211,12 @@ int main(int argc, char *argv[]) args.AddOption(&gpu_aware_mpi, "-gam", "--gpu-aware-mpi", "-no-gam", "--no-gpu-aware-mpi", "Enable GPU aware MPI communications."); args.AddOption(&dev, "-dev", "--dev", "GPU device to use."); + args.AddOption(&amr, "-amr", "--enable-amr", "-no-amr", "--disable-amr", "Experimental adaptive mesh refinement (problem 1 only)."); - args.AddOption(&amr_ref_threshold, "-rt", "--ref-threshold", + args.AddOption(&amr_ref_threshold, "-ar", "--amr-ref-threshold", "AMR refinement threshold."); - args.AddOption(&amr_deref_threshold, "-dt", "--deref-threshold", + args.AddOption(&amr_deref_threshold, "-ad", "--amr-deref-threshold", "AMR derefinement threshold (0 = no derefinement)."); args.Parse(); if (!args.Good()) @@ -238,9 +239,9 @@ int main(int argc, char *argv[]) if (mpi.Root()) { backend.Print(); } backend.SetGPUAwareMPI(gpu_aware_mpi); - // On all processors, use the default builtin 1D/2D/3D mesh or read the - // serial one given on the command line. - Mesh *mesh; + // On all processors, use the default builtin 1D/2D/3D mesh + // or read the serial one given on the command line. + Mesh *mesh = nullptr; if (strncmp(mesh_file, "default", 7) != 0) { mesh = new Mesh(mesh_file, true, true); @@ -295,7 +296,6 @@ int main(int argc, char *argv[]) } else { - dbg("Initial refinement for AMR demo on Sedov"); mesh->EnsureNCMesh(); for (int lev = 0; lev < rs_levels; lev++) { @@ -502,14 +502,12 @@ int main(int argc, char *argv[]) // - 2 -> specific internal energy const int Vsize_l2 = L2FESpace.GetVSize(); const int Vsize_h1 = H1FESpace.GetVSize(); - dbg("Init: Vsize_h1:%d Vsize_l2:%d", Vsize_h1,Vsize_l2); Array true_offset(4); true_offset[0] = 0; true_offset[1] = true_offset[0] + Vsize_h1; true_offset[2] = true_offset[1] + Vsize_h1; true_offset[3] = true_offset[2] + Vsize_l2; BlockVector S(true_offset, Device::GetMemoryType()); - dbg("Init: S:%d", S.Size()); // Define GridFunction objects for the position, velocity and specific // internal energy. There is no function for the density, as we can always @@ -570,9 +568,9 @@ int main(int argc, char *argv[]) // gamma values are projected on function that's constant on the moving mesh. L2_FECollection mat_fec(0, pmesh->Dimension()); ParFiniteElementSpace mat_fes(pmesh, &mat_fec); - ParGridFunction mat_gf(&mat_fes); + ParGridFunction m_gf(&mat_fes); FunctionCoefficient mat_coeff(gamma_func); - mat_gf.ProjectCoefficient(mat_coeff); + m_gf.ProjectCoefficient(mat_coeff); // Additional details, depending on the problem. int source = 0; bool visc = true, vorticity = false; @@ -594,7 +592,7 @@ int main(int argc, char *argv[]) H1FESpace, L2FESpace, ess_tdofs, rho0_coeff, rho0_gf, - mat_gf, source, cfl, + m_gf, source, cfl, visc, vorticity, p_assembly, amr, cg_tol, cg_max_iter, ftz_tol, @@ -602,7 +600,7 @@ int main(int argc, char *argv[]) /*if (amr) { - dbg("set a base for h0, this will be further divided in UpdateQuadratureData"); + // Set a base for h0, this will be further divided in UpdateQuadratureData // TODO: for AMR, the treatment of h0 needs more work const double elem_size = 0.5; // coarse element size (TODO calculate) const double h0 = elem_size / order_v; @@ -628,19 +626,18 @@ int main(int argc, char *argv[]) vis_e.precision(8); int Wx = 0, Wy = 0; // window position const int Ww = 350, Wh = 350; // window size - //int offx = Ww+10; // window offsets + int offx = Ww + 10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); - }/* + } Wx += offx; hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, "Velocity", Wx, Wy, Ww, Wh); Wx += offx; hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", Wx, Wy, Ww, Wh); -*/ } // Save data for VisIt visualization. @@ -686,7 +683,6 @@ int main(int argc, char *argv[]) // Adaptive time step control. const double dt_est = hydro.GetTimeStepEstimate(S); - dbg("dt_est:%.8e, dt:%.8e", dt_est,dt); if (dt_est < dt) { // Repeat (solve again) with a decreased time step - decrease of the @@ -751,12 +747,12 @@ int main(int argc, char *argv[]) { int Wx = 0, Wy = 0; // window position int Ww = 350, Wh = 350; // window size - //int offx = Ww+10; // window offsets + int offx = Ww + 10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); - }/* + } Wx += offx; hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, "Velocity", Wx, Wy, Ww, Wh); @@ -764,17 +760,17 @@ int main(int argc, char *argv[]) hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", Wx, Wy, Ww,Wh); - Wx += offx;*/ + Wx += offx; } - /*if (visit) + if (visit) { visit_dc.SetCycle(ti); visit_dc.SetTime(t); visit_dc.Save(); - }*/ + } - /*if (gfprint) + if (gfprint) { std::ostringstream mesh_name, rho_name, v_name, e_name; mesh_name << basename << "_" << ti << "_mesh"; @@ -801,15 +797,14 @@ int main(int argc, char *argv[]) e_ofs.precision(8); e_gf.SaveAsOne(e_ofs); e_ofs.close(); - }*/ + } } // last_step if (amr) { - Vector &error_est = hydro.GetZoneMaxVisc(); - Vector v_max, v_min; GetPerElementMinMax(v_gf, v_min, v_max); + Vector &error_est = hydro.GetZoneMaxVisc(); bool mesh_changed = false; @@ -823,7 +818,6 @@ int main(int argc, char *argv[]) ) { refs.Append(i); - dbg("Refine #%d",i); } } @@ -844,7 +838,7 @@ int main(int argc, char *argv[]) Vector rho_max, rho_min; GetPerElementMinMax(rho_gf, rho_min, rho_max); - // simple derefinement based on zone maximum rho in post-shock region + // Derefinement based on zone maximum rho in post-shock region const double rho_max_max = rho_max.Size() ? rho_max.Max() : 0.0; double threshold, loc_threshold = amr_deref_threshold * rho_max_max; MPI_Allreduce(&loc_threshold, &threshold, 1, MPI_DOUBLE, MPI_MAX, @@ -862,7 +856,7 @@ int main(int argc, char *argv[]) if (index >= 0) { rho_max(index) = 1e10; } } - // also, only derefine where the mesh is in motion, i.e. after the shock + // only derefine where the mesh is in motion, i.e. after the shock for (int i = 0; i < pmesh->GetNE(); i++) { if (v_min(i) < 0.1) { rho_max(i) = 1e10; } @@ -879,44 +873,19 @@ int main(int argc, char *argv[]) if (mesh_changed) { - // update state and operator - { - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - mat_fes.Update(); - mat_gf.Update(); - mat_gf.ProjectCoefficient(mat_coeff); - } + constexpr bool quick = true; - hydro.UpdateMesh(S); - hydro.AMRUpdate(S, true); // quick + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); + hydro.AMRUpdate(S, quick); pmesh->Rebalance(); - // update state and operator - { - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - mat_fes.Update(); - mat_gf.Update(); - mat_gf.ProjectCoefficient(mat_coeff); - } - - hydro.UpdateMesh(S); - hydro.AMRUpdate(S, false); // thorough + AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); + hydro.AMRUpdate(S, !quick); GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); - /*{ - v_gf.ProjectCoefficient(v_coeff); - for (int i = 0; i < ess_vdofs.Size(); i++) - { - v_gf(ess_vdofs[i]) = 0.0; - } - // Sync the data location of v_gf with its base, S - v_gf.SyncAliasMemory(S); - }*/ - ode_solver->Init(hydro); - - H1FESpace.PrintPartitionStats(); + //H1FESpace.PrintPartitionStats(); } } @@ -1027,13 +996,16 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, Array &true_offset, ParGridFunction &x_gf, ParGridFunction &v_gf, - ParGridFunction &e_gf) + ParGridFunction &e_gf, + ParGridFunction &m_gf) { ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); H1FESpace->Update(); L2FESpace->Update(); + MEFESpace->Update(); const int Vsize_h1 = H1FESpace->GetVSize(); const int Vsize_l2 = L2FESpace->GetVSize(); @@ -1059,6 +1031,7 @@ void AMRUpdate(BlockVector &S, BlockVector &S_tmp, e_gf.SyncAliasMemory(S); S_tmp.Update(true_offset); + m_gf.Update(); } void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index 6dec3ab6..60ee07f3 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -17,9 +17,6 @@ #include "laghos_assembly.hpp" #include -#define MFEM_DEBUG_COLOR 226 -#include "general/debug.hpp" - namespace mfem { diff --git a/laghos_solver.cpp b/laghos_solver.cpp index 99a6bbac..2bab1f86 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -19,9 +19,6 @@ #include "linalg/kernels.hpp" #include -#define MFEM_DEBUG_COLOR 198 -#include "general/debug.hpp" - #ifdef MFEM_USE_MPI namespace mfem @@ -313,74 +310,37 @@ LagrangianHydroOperator::~LagrangianHydroOperator() void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) { - dbg("%s AMR update", quick ? "QUICK" : "THOROUGH"); - dbg("S:%d, NE:%d", width, NE); width = height = S.Size(); + H1c.Update(); pmesh = H1.GetParMesh(); - pmesh->DeleteGeometricFactors(); H1Vsize = H1.GetVSize(); H1TVSize = H1.TrueVSize(); H1GTVSize = H1.GlobalTrueVSize(); L2Vsize = L2.GetVSize(); L2TVSize = L2.TrueVSize(); L2GTVSize = L2.GlobalTrueVSize(); - block_offsets[0] = 0; block_offsets[1] = block_offsets[0] + H1Vsize; block_offsets[2] = block_offsets[1] + H1Vsize; block_offsets[3] = block_offsets[2] + L2Vsize; - this->x_gf.Update(); - // ess_tdofs NE = pmesh->GetNE(); - // dim l2dofs_cnt = (L2.GetFE(0)->GetDof()); h1dofs_cnt = (H1.GetFE(0)->GetDof()); - // source_type - // cfl - // use_viscosity - // use_vorticity - // p_assembly - // amr - // cg_rel_tol - // cg_max_iter - // ftz_tol - gamma_gf.Update(); rho0_gf.Update(); x0_gf.Update(); - // Mv_spmat_copy - //Me.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - //Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, NE); - // ir - // Q1D qdata.Resize(dim, NE, ir.GetNPoints()); qdata_is_current = false; forcemat_is_assembled = false; - Force.Update(); delete ForcePA; ForcePA = nullptr; delete VMassPA; VMassPA = nullptr; delete EMassPA; EMassPA = nullptr; delete VMassPA_Jprec; VMassPA_Jprec = nullptr; - // CG_VMass - // CG_EMass - // timer // to update with new sizes delete qupdate; qupdate = nullptr; - X.SetSize(H1c.GetTrueVSize()); - B.SetSize(H1c.GetTrueVSize()); - one.SetSize(L2Vsize); - one.UseDevice(true); one = 1.0; - rhs.SetSize(H1Vsize); e_rhs.SetSize(L2Vsize); - H1c.Update(); - //rhs_c_gf.Update(); - //dvc_gf.Update(); - // c_tdofs - // zone_max_visc, zone_vgrad - - /////////////////////// if (quick) { return; } // go back to initial mesh configuration temporarily @@ -390,6 +350,8 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) if (p_assembly) { + gamma_gf.Update(); + pmesh->DeleteGeometricFactors(); qupdate = new QUpdate(dim, NE, Q1D, use_viscosity, use_vorticity, cfl, &timer, gamma_gf, ir, H1, L2); ForcePA = new ForcePAOperator(qdata, H1, L2, ir); @@ -411,16 +373,37 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) H1c.GetEssentialTrueDofs(ess_bdr, c_tdofs[c]); c_tdofs[c].Read(); } + X.SetSize(H1c.GetTrueVSize()); + B.SetSize(H1c.GetTrueVSize()); X.UseDevice(true); B.UseDevice(true); rhs.UseDevice(true); e_rhs.UseDevice(true); + + // Setup the preconditioner of the velocity mass operator. + // BC are handled by the VMassPA, so ess_tdofs here can be empty. + Array ess_tdofs; + VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); + CG_VMass.SetPreconditioner(*VMassPA_Jprec); + + CG_VMass.SetOperator(*VMassPA); + CG_VMass.SetRelTol(cg_rel_tol); + CG_VMass.SetAbsTol(0.0); + CG_VMass.SetMaxIter(cg_max_iter); + CG_VMass.SetPrintLevel(-1); + + CG_EMass.SetOperator(*EMassPA); + CG_EMass.iterative_mode = false; + CG_EMass.SetRelTol(cg_rel_tol); + CG_EMass.SetAbsTol(0.0); + CG_EMass.SetMaxIter(cg_max_iter); + CG_EMass.SetPrintLevel(-1); } else { - // update mass matrix - // TODO: do this better too - // TODO: don't reassemble everything! + Force.Update(); + Force.Assemble(0); + Force.Finalize(0); Mv.Update(); Mv.Assemble(); Mv_spmat_copy = Mv.SpMat(); @@ -436,14 +419,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) inv.Factor(); inv.GetInverseMatrix(Me_inv(e)); } - // Standard assembly for the velocity mass matrix. - //VectorMassIntegrator *vmi = new VectorMassIntegrator(rho0_coeff, &ir); - //Mv.AddDomainIntegrator(vmi); - //Mv.Assemble(); - //Mv_spmat_copy = Mv.SpMat(); } - - // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points // TODO: remove code duplication int Ne, ne = NE; @@ -484,46 +460,13 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) default: MFEM_ABORT("Unknown zone type!"); } qdata.h0 /= (double) H1.GetOrder(0); - dbg("h0:%.15e", qdata.h0); // swap back to deformed mesh configuration pmesh->SwapNodes(x_gf, own_nodes); - - if (p_assembly) - { - // Setup the preconditioner of the velocity mass operator. - // BC are handled by the VMassPA, so ess_tdofs here can be empty. - Array ess_tdofs; - VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); - CG_VMass.SetPreconditioner(*VMassPA_Jprec); - - CG_VMass.SetOperator(*VMassPA); - CG_VMass.SetRelTol(cg_rel_tol); - CG_VMass.SetAbsTol(0.0); - CG_VMass.SetMaxIter(cg_max_iter); - CG_VMass.SetPrintLevel(-1); - - CG_EMass.SetOperator(*EMassPA); - CG_EMass.iterative_mode = false; - CG_EMass.SetRelTol(cg_rel_tol); - CG_EMass.SetAbsTol(0.0); - CG_EMass.SetMaxIter(cg_max_iter); - CG_EMass.SetPrintLevel(-1); - } - else - { - //ForceIntegrator *fi = new ForceIntegrator(qdata); - //fi->SetIntRule(&ir); - //Force.AddDomainIntegrator(fi); - // Make a dummy assembly to figure out the sparsity. - Force.Assemble(0); - Force.Finalize(0); - } } void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const { - dbg("S:%.15e", S*S); // Make sure that the mesh positions correspond to the ones in S. This is // needed only because some mfem time integrators don't update the solution // vector at every intermediate stage (hence they don't change the mesh). @@ -534,16 +477,12 @@ void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const ParGridFunction v; const int VsizeH1 = H1.GetVSize(); v.MakeRef(&H1, *sptr, VsizeH1); - dbg("v:%.15e", v*v); // Set dx_dt = v (explicit). ParGridFunction dx; dx.MakeRef(&H1, dS_dt, 0); dx = v; - dbg("dx:%.15e", dx*dx); SolveVelocity(S, dS_dt); - dbg("dS_dt:%.15e", dS_dt*dS_dt); SolveEnergy(S, v, dS_dt); - dbg("dS_dt:%.15e", dS_dt*dS_dt); qdata_is_current = false; } @@ -642,7 +581,6 @@ void LagrangianHydroOperator::SolveVelocity(const Vector &S, timer.H1iter += cg.GetNumIterations(); Mv.RecoverFEMSolution(X, rhs, dv); } - dbg("S:%.15e dv:%.15e", S*S, dv*dv); } void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, @@ -702,7 +640,6 @@ void LagrangianHydroOperator::SolveEnergy(const Vector &S, const Vector &v, de.SetSubVector(l2dofs, loc_de); } } - dbg("v:%.15e de:%.15e", v*v, de*de); delete e_source; } @@ -1301,11 +1238,6 @@ static void Rho0DetJ0Vol(const int dim, const int NE, const double J21 = J(q,0,1,e); const double J22 = J(q,1,1,e); const double det = detJ(q,e); - /*if (e==0 && q == 0) - { - dbg("%.15e %.15e %.15e %.15e = %.15e", - J11, J12, J21, J22, det); - }*/ V(q,e) = W[q] * R(q,e) * det; const double r_idetJ = 1.0 / det; invJ(0,0,q,e) = J22 * r_idetJ; From 9c5737546f97c7a164ed4808a59721c0d25a11c1 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 17:39:43 -0800 Subject: [PATCH 08/22] Add geometric refinment estimator --- laghos.cpp | 175 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 107 insertions(+), 68 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 096d1970..06cc5e3b 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -64,6 +64,9 @@ #include #include "laghos_solver.hpp" +#define MFEM_DEBUG_COLOR 199 +#include "general/debug.hpp" + using std::cout; using std::endl; using namespace mfem; @@ -138,8 +141,11 @@ int main(int argc, char *argv[]) double blast_energy = 0.25; double blast_position[] = {0.0, 0.0, 0.0}; bool amr = false; + int amr_estimator = 0; double amr_ref_threshold = 2e-4; + double amr_jac_threshold = 0.92; double amr_deref_threshold = 0.75; + int amr_max_level = rs_levels + rp_levels; const double amr_blast_size = 1e-10; const int amr_nc_limit = 1; @@ -216,8 +222,14 @@ int main(int argc, char *argv[]) "Experimental adaptive mesh refinement (problem 1 only)."); args.AddOption(&amr_ref_threshold, "-ar", "--amr-ref-threshold", "AMR refinement threshold."); + args.AddOption(&amr_jac_threshold, "-aj", "--amr-jac-threshold", + "AMR Jacobian refinement threshold."); args.AddOption(&amr_deref_threshold, "-ad", "--amr-deref-threshold", "AMR derefinement threshold (0 = no derefinement)."); + args.AddOption(&amr_estimator, "-ae", "--amr-estimator", + "AMR estimator: 0:vgrad/visc 1:Jacobians"); + args.AddOption(&amr_max_level, "-am", "--amr-max-level", + "AMR max refined level (default to 'rs_levels + rp_levels')"); args.Parse(); if (!args.Good()) { @@ -225,13 +237,15 @@ int main(int argc, char *argv[]) return 1; } + amr_max_level = std::max(amr_max_level, rs_levels + rp_levels); + if (mpi.Root()) { args.PrintOptions(cout); } - if (amr && problem != 1) + /*if (amr && problem != 1) { if (mpi.Root()) { cout << "AMR only supported for problem 1." << endl; } return 0; - } + }*/ // Configure the device from the command line options Device backend; @@ -254,7 +268,7 @@ int main(int argc, char *argv[]) mesh->GetBdrElement(0)->SetAttribute(1); mesh->GetBdrElement(1)->SetAttribute(1); } - if (dim == 2) + else if (dim == 2) { mesh = new Mesh(2, 2, Element::QUADRILATERAL, true); const int NBE = mesh->GetNBE(); @@ -265,7 +279,7 @@ int main(int argc, char *argv[]) bel->SetAttribute(attr); } } - if (dim == 3) + else if (dim == 3) { mesh = new Mesh(2, 2, 2, Element::HEXAHEDRON, true); const int NBE = mesh->GetNBE(); @@ -276,6 +290,7 @@ int main(int argc, char *argv[]) bel->SetAttribute(attr); } } + else { MFEM_ABORT("Dimension should be set"); } } dim = mesh->Dimension(); @@ -299,9 +314,8 @@ int main(int argc, char *argv[]) mesh->EnsureNCMesh(); for (int lev = 0; lev < rs_levels; lev++) { - mesh->RefineAtVertex(Vertex(blast_position[0], blast_position[1], - blast_position[2]), - amr_blast_size); + Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; + mesh->RefineAtVertex(blast, amr_blast_size); } } @@ -404,7 +418,7 @@ int main(int argc, char *argv[]) } int product = 1; for (int d = 0; d < dim; d++) { product *= nxyz[d]; } - const bool cartesian_partitioning = (cxyz.Size()>0)?true:false; + const bool cartesian_partitioning = (cxyz.Size() > 0)?true:false; if (product == num_tasks || cartesian_partitioning) { if (cartesian_partitioning) @@ -450,8 +464,6 @@ int main(int argc, char *argv[]) if (myid == 0) { cout << "Zones min/max: " << ne_min << " " << ne_max << endl; } - const int amr_max_level = rs_levels + rp_levels; - // Define the parallel finite element spaces. We use: // - H1 (Gauss-Lobatto, continuous) for position and velocity. // - L2 (Bernstein, discontinuous) for specific internal energy. @@ -523,14 +535,10 @@ int main(int argc, char *argv[]) // Sync the data location of x_gf with its base, S x_gf.SyncAliasMemory(S); - // Initialize the velocity. + // Initialize the velocity and sync the data of v_gf with its base, S VectorFunctionCoefficient v_coeff(pmesh->Dimension(), v0); v_gf.ProjectCoefficient(v_coeff); - for (int i = 0; i < ess_vdofs.Size(); i++) - { - v_gf(ess_vdofs[i]) = 0.0; - } - // Sync the data location of v_gf with its base, S + for (int i = 0; i < ess_vdofs.Size(); i++) { v_gf(ess_vdofs[i]) = 0.0; } v_gf.SyncAliasMemory(S); // Initialize density and specific internal energy values. We interpolate in @@ -598,15 +606,6 @@ int main(int argc, char *argv[]) cg_tol, cg_max_iter, ftz_tol, order_q); - /*if (amr) - { - // Set a base for h0, this will be further divided in UpdateQuadratureData - // TODO: for AMR, the treatment of h0 needs more work - const double elem_size = 0.5; // coarse element size (TODO calculate) - const double h0 = elem_size / order_v; - hydro.SetH0(h0); - }*/ - socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; int visport = 19916; @@ -663,27 +662,27 @@ int main(int argc, char *argv[]) BlockVector S_old(S); long mem = 0, mmax = 0, msum = 0; int checks = 0; -// const double internal_energy = hydro.InternalEnergy(e_gf); -// const double kinetic_energy = hydro.KineticEnergy(v_gf); -// if (mpi.Root()) -// { -// cout << std::fixed; -// cout << "step " << std::setw(5) << 0 -// << ",\tt = " << std::setw(5) << std::setprecision(4) << t -// << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt -// << ",\t|IE| = " << std::setprecision(10) << std::scientific -// << internal_energy -// << ",\t|KE| = " << std::setprecision(10) << std::scientific -// << kinetic_energy -// << ",\t|E| = " << std::setprecision(10) << std::scientific -// << kinetic_energy+internal_energy; -// cout << std::fixed; -// if (mem_usage) -// { -// cout << ", mem: " << mmax << "/" << msum << " MB"; -// } -// cout << endl; -// } + // const double internal_energy = hydro.InternalEnergy(e_gf); + // const double kinetic_energy = hydro.KineticEnergy(v_gf); + // if (mpi.Root()) + // { + // cout << std::fixed; + // cout << "step " << std::setw(5) << 0 + // << ",\tt = " << std::setw(5) << std::setprecision(4) << t + // << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt + // << ",\t|IE| = " << std::setprecision(10) << std::scientific + // << internal_energy + // << ",\t|KE| = " << std::setprecision(10) << std::scientific + // << kinetic_energy + // << ",\t|E| = " << std::setprecision(10) << std::scientific + // << kinetic_energy+internal_energy; + // cout << std::fixed; + // if (mem_usage) + // { + // cout << ", mem: " << mmax << "/" << msum << " MB"; + // } + // cout << endl; + // } for (int ti = 1; !last_step; ti++) { if (t + dt >= t_final) @@ -753,12 +752,6 @@ int main(int argc, char *argv[]) << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt << ",\t|e| = " << std::setprecision(10) << std::scientific << sqrt_norm; - // << ",\t|IE| = " << std::setprecision(10) << std::scientific - // << internal_energy - // << ",\t|KE| = " << std::setprecision(10) << std::scientific - // << kinetic_energy - // << ",\t|E| = " << std::setprecision(10) << std::scientific - // << kinetic_energy+internal_energy; cout << std::fixed; if (mem_usage) { @@ -789,7 +782,6 @@ int main(int argc, char *argv[]) hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", Wx, Wy, Ww,Wh); - Wx += offx; } if (visit) @@ -832,35 +824,82 @@ int main(int argc, char *argv[]) if (amr) { Vector v_max, v_min; - GetPerElementMinMax(v_gf, v_min, v_max); - Vector &error_est = hydro.GetZoneMaxVisc(); - + Array refs; bool mesh_changed = false; + const int NE = pmesh->GetNE(); + constexpr double NL_DMAX = std::numeric_limits::max(); - // make a list of elements to refine - Array refs; - for (int i = 0; i < pmesh->GetNE(); i++) + GetPerElementMinMax(v_gf, v_min, v_max); + + switch (amr_estimator) { - if (error_est(i) > amr_ref_threshold - && pmesh->pncmesh->GetElementDepth(i) < amr_max_level - && (v_min(i) < 1e-3 /*|| ti < 50*/) // only refine the still area - ) + case 0: + { + Vector &error_est = hydro.GetZoneMaxVisc(); + for (int e = 0; e < NE; e++) + { + if (error_est(e) > amr_ref_threshold && + pmesh->pncmesh->GetElementDepth(e) < amr_max_level && + (v_min(e) < 1e-3)) // only refine the still area + { + refs.Append(Refinement(e)); + } + } + break; + } + case 1: { - refs.Append(i); + const double art = amr_ref_threshold; + const int order = H1FESpace.GetOrder(0) + 1; + DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); + MFEM_VERIFY(art >= 0.0, "AMR threshold should be positive"); + for (int e = 0; e < NE; e++) + { + double minW = +NL_DMAX; + double maxW = -NL_DMAX; + const int depth = pmesh->pncmesh->GetElementDepth(e); + ElementTransformation *eTr = pmesh->GetElementTransformation(e); + const Geometry::Type &type = pmesh->GetElement(e)->GetGeometryType(); + const IntegrationRule *ir = &IntRules.Get(type, order); + const int NQ = ir->GetNPoints(); + for (int q = 0; q < NQ; q++) + { + eTr->SetIntPoint(&ir->IntPoint(q)); + const DenseMatrix &J = eTr->Jacobian(); + CalcAdjugate(J, Jadj); + Jadjt = Jadj; + Jadjt.Transpose(); + const double w = Jadjt.Weight(); + minW = std::fmin(minW, w); + maxW = std::fmax(maxW, w); + } + if (std::fabs(maxW) != 0.0) + { + const double rho = minW / maxW; + MFEM_VERIFY(rho <= 1.0, ""); + if (rho < amr_jac_threshold && depth < amr_max_level) + { + refs.Append(Refinement(e)); + } + } + } + break; } + default: MFEM_ABORT("Unknown AMR estimator (should be 0 or 1)!"); } const int nref = pmesh->ReduceInt(refs.Size()); if (nref) { - pmesh->GeneralRefinement(refs, 1, amr_nc_limit); + constexpr int non_conforming = 1; + pmesh->GeneralRefinement(refs, non_conforming, amr_nc_limit); mesh_changed = true; if (myid == 0) { std::cout << "Refined " << nref << " elements." << std::endl; } } - else if (amr_deref_threshold) + else if (amr_estimator == 0 && amr_deref_threshold >= 0.0) { hydro.ComputeDensity(rho_gf); @@ -882,13 +921,13 @@ int main(int argc, char *argv[]) for (int i = 0; i < elements.Size(); i++) { int index = elements[i]; - if (index >= 0) { rho_max(index) = 1e10; } + if (index >= 0) { rho_max(index) = NL_DMAX; } } // only derefine where the mesh is in motion, i.e. after the shock for (int i = 0; i < pmesh->GetNE(); i++) { - if (v_min(i) < 0.1) { rho_max(i) = 1e10; } + if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } } const int op = 2; // maximum value of fine elements From 443bd09ab4a3d2f5cff9f01621a65fa5136519f0 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 12 Jan 2021 17:51:26 -0800 Subject: [PATCH 09/22] Remove debug traces --- laghos.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 06cc5e3b..bd177bcc 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -64,9 +64,6 @@ #include #include "laghos_solver.hpp" -#define MFEM_DEBUG_COLOR 199 -#include "general/debug.hpp" - using std::cout; using std::endl; using namespace mfem; From 27afcebd102cfd3172b40adb0643cc26951aa055 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 13 Jan 2021 08:38:43 -0800 Subject: [PATCH 10/22] Update README.md with AMR input from ./amr --- README.md | 22 + amr/README.md | 123 --- amr/laghos.cpp | 959 ---------------------- amr/laghos_assembly.cpp | 1072 ------------------------- amr/laghos_assembly.hpp | 230 ------ amr/laghos_solver.cpp | 737 ----------------- amr/laghos_solver.hpp | 191 ----- amr/makefile | 183 ----- {amr/data => data}/sedov-amr-2463.png | Bin {amr/data => data}/sedov-amr-900.png | Bin 10 files changed, 22 insertions(+), 3495 deletions(-) delete mode 100644 amr/README.md delete mode 100644 amr/laghos.cpp delete mode 100644 amr/laghos_assembly.cpp delete mode 100644 amr/laghos_assembly.hpp delete mode 100644 amr/laghos_solver.cpp delete mode 100644 amr/laghos_solver.hpp delete mode 100644 amr/makefile rename {amr/data => data}/sedov-amr-2463.png (100%) rename {amr/data => data}/sedov-amr-900.png (100%) diff --git a/README.md b/README.md index 9bc7e43e..b3d0cad3 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ Other computational motives in Laghos include the following: partially assembled) and is applied just twice per "assembly". Both the preparation and the application costs are important for this operator. - Domain-decomposed MPI parallelism. +- Automatic mesh refinement (AMR) with parallel partitioning and load balancing + based on MFEM's non-conforming mesh algorithm that partitions a space-filling + curve. Only the Sedov problem (#1) is supported. - Optional in-situ visualization with [GLVis](http:/glvis.org) and data output for visualization and data analysis with [VisIt](http://visit.llnl.gov). @@ -250,6 +253,25 @@ The latter produces the following specific internal energy plot (notice the `-vi ![Triple-point image](data/tp.png) +#### AMR Sedov problem +The AMR version only runs with problem 1 (Sedov blast). New parameters are: + +- `-amr`: turn on AMR mode +- `-rt` or `--ref-threshold`: tweak the refinement threshold +- `-dt` or `--deref-threshold`: tweak the derefinement threshold + +One of the sample runs is: +```sh +mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 4 -tf 0.6 -rt 1e-3 -amr +``` + +This produces the following plots: + + +
+ +
+ ## Verification of Results To make sure the results are correct, we tabulate reference final iterations diff --git a/amr/README.md b/amr/README.md deleted file mode 100644 index 13f06548..00000000 --- a/amr/README.md +++ /dev/null @@ -1,123 +0,0 @@ - __ __ - / / ____ ____ / /_ ____ _____ - / / / __ `/ __ `/ __ \/ __ \/ ___/ - / /___/ /_/ / /_/ / / / / /_/ (__ ) - /_____/\__,_/\__, /_/ /_/\____/____/ - /____/ - - High-order Lagrangian Hydrodynamics Miniapp - - AMR version - - -## Overview - -This directory contains the automatic mesh refinement (AMR) version of **Laghos** -(LAGrangian High-Order Solver), which is currently considered experimental and -is NOT the official benchmark version of the miniapp. - -For more details about Laghos see the [README file](../README.md) in the -top-level directory. - -The Laghos miniapp is part of the [CEED software suite](http://ceed.exascaleproject.org/software), -a collection of software benchmarks, miniapps, libraries and APIs for -efficient exascale discretizations based on high-order finite element -and spectral element methods. See http://github.com/ceed for more -information and source code availability. - -The CEED research is supported by the [Exascale Computing Project](https://exascaleproject.org/exascale-computing-project) -(17-SC-20-SC), a collaborative effort of two U.S. Department of Energy -organizations (Office of Science and the National Nuclear Security -Administration) responsible for the planning and preparation of a -[capable exascale ecosystem](https://exascaleproject.org/what-is-exascale), -including software, applications, hardware, advanced system engineering and early -testbed platforms, in support of the nation’s exascale computing imperative. - - -## Differences from the official benchmark version - -The AMR version differs from the official benchmark version of Laghos (in the -top-level directory) in the following ways: - -1. The `-amr` parameter turns on dynamic AMR. -2. The code includes functionality to change the mesh and the hydro operator on - the fly. -3. Parallel partitioning and load balancing is based on MFEM's non-conforming - mesh algorithm that partitions a space-filling curve. METIS is not required. - - -## Limitations - -- The current AMR implementation is just a demonstration. -- Only the Sedov problem is supported, as the refinement/derefinement decisions - are very simple and tailored specifically to Sedov. -- Partial assembly is currently not supported in AMR mode. Also, the hydro - operator update is currently not efficient (e.g., the whole mass matrix is - reassembled on each mesh change). -- MFEM currently does not support derefinement interpolation for non-nodal bases. - The AMR version therefore does not use `BasisType::Positive` for the L2 space. - - -## Building - -The AMR version can be built following the same [instructions](../README.md) as -for the top-level directory. - - -## Running - -The AMR version only runs with problem 1 (Sedov blast). New parameters are: - -- `-amr`: turn on AMR mode -- `-rt` or `--ref-threshold`: tweak the refinement threshold -- `-dt` or `--deref-threshold`: tweak the derefinement threshold - -One of the sample runs is: -```sh -mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 4 -tf 0.6 -rt 1e-3 -amr -``` - -This produces the following plots at steps 900 and 2463: - - -
- -
- - -## Verification of Results - -To make sure the results are correct, we tabulate reference final iterations -(`step`), time steps (`dt`) and energies (`|e|`) for the runs listed below: - -1. `mpirun -np 8 laghos -p 1 -m ../data/square01_quad.mesh -rs 4 -tf 0.8 -amr` -2. `mpirun -np 8 laghos -p 1 -m ../data/square01_quad.mesh -rs 4 -tf 0.8 -ok 3 -ot 2 -amr` -3. `mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 3 -tf 0.6 -amr` -4. `mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 4 -tf 0.6 -rt 1e-3 -amr` - -| run | `step` | `dt` | `e` | -| --- | ------ | ---- | ----- | -| 1. | 2374 | 0.000308 | 90.9397751791 | -| 2. | 2727 | 0.000458 | 168.0063715464 | -| 3. | 998 | 0.001262 | 388.6322346715 | -| 4. | 2463 | 0.000113 | 1703.2772575684 | - -An implementation is considered valid if the final energy values are all within -round-off distance from the above reference values. - - -## Contact - -You can reach the Laghos team by emailing laghos@llnl.gov or by leaving a -comment in the [issue tracker](https://github.com/CEED/Laghos/issues). - - -## Copyright - -The following copyright applies to each file in the CEED software suite, -unless otherwise stated in the file: - -> Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at the -> Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights reserved. - -See files LICENSE and NOTICE in the top-level directory for details. diff --git a/amr/laghos.cpp b/amr/laghos.cpp deleted file mode 100644 index 94fa9b02..00000000 --- a/amr/laghos.cpp +++ /dev/null @@ -1,959 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. -// -// __ __ -// / / ____ ____ / /_ ____ _____ -// / / / __ `/ __ `/ __ \/ __ \/ ___/ -// / /___/ /_/ / /_/ / / / / /_/ (__ ) -// /_____/\__,_/\__, /_/ /_/\____/____/ -// /____/ -// -// High-order Lagrangian Hydrodynamics Miniapp -// -// Laghos(LAGrangian High-Order Solver) is a miniapp that solves the -// time-dependent Euler equation of compressible gas dynamics in a moving -// Lagrangian frame using unstructured high-order finite element spatial -// discretization and explicit high-order time-stepping. Laghos is based on the -// numerical algorithm described in the following article: -// -// V. Dobrev, Tz. Kolev and R. Rieben, "High-order curvilinear finite element -// methods for Lagrangian hydrodynamics", SIAM Journal on Scientific -// Computing, (34) 2012, pp. B606–B641, https://doi.org/10.1137/120864672. -// -// *** THIS IS AN AUTOMATIC MESH REFINEMENT DEMO *** -// -// Sample runs: -// mpirun -np 8 laghos -p 1 -m ../data/square01_quad.mesh -rs 4 -tf 0.8 -amr -// mpirun -np 8 laghos -p 1 -m ../data/square01_quad.mesh -rs 4 -tf 0.8 -ok 3 -ot 2 -amr -// mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 3 -tf 0.6 -amr -// mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 4 -tf 0.6 -rt 1e-3 -amr -// -// Test problems: -// p = 1 --> Sedov blast. - - -#include "laghos_solver.hpp" -#include -#include -#include - -using namespace std; -using namespace mfem; -using namespace mfem::hydrodynamics; - -// Choice for the problem setup. -int problem; - -void display_banner(ostream & os); - -void AMRUpdate(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf); - -void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace *pspace, - int bdr_attr_max, Array &ess_tdofs); - -void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements); - -void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order = -1); - - -int main(int argc, char *argv[]) -{ - // Initialize MPI. - MPI_Session mpi(argc, argv); - int myid = mpi.WorldRank(); - - // Print the banner. - if (mpi.Root()) { display_banner(cout); } - - // Parse command-line options. - const char *mesh_file = "data/square01_quad.mesh"; - int rs_levels = 0; - int rp_levels = 0; - int order_v = 2; - int order_e = 1; - int ode_solver_type = 4; - double t_final = 0.5; - double cfl = 0.5; - double cg_tol = 1e-8; - int cg_max_iter = 300; - int max_tsteps = -1; - bool p_assembly = true; - bool visualization = false; - int vis_steps = 5; - bool visit = false; - bool gfprint = false; - const char *basename = "results/Laghos"; - int partition_type = 111; - bool amr = false; - double ref_threshold = 2e-4; - double deref_threshold = 0.75; - const int nc_limit = 1; - const double blast_energy = 0.25; - const double blast_position[] = {0.0, 0.0, 0.0}; - const double blast_amr_size = 1e-10; - - OptionsParser args(argc, argv); - args.AddOption(&mesh_file, "-m", "--mesh", - "Mesh file to use."); - args.AddOption(&rs_levels, "-rs", "--refine-serial", - "Number of times to refine the mesh uniformly in serial."); - args.AddOption(&rp_levels, "-rp", "--refine-parallel", - "Number of times to refine the mesh uniformly in parallel."); - - args.AddOption(&problem, "-p", "--problem", "Problem setup to use."); - args.AddOption(&order_v, "-ok", "--order-kinematic", - "Order (degree) of the kinematic finite element space."); - args.AddOption(&order_e, "-ot", "--order-thermo", - "Order (degree) of the thermodynamic finite element space."); - args.AddOption(&ode_solver_type, "-s", "--ode-solver", - "ODE solver: 1 - Forward Euler,\n\t" - " 2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6."); - args.AddOption(&t_final, "-tf", "--t-final", - "Final time; start time is 0."); - - args.AddOption(&cfl, "-cfl", "--cfl", "CFL-condition number."); - args.AddOption(&cg_tol, "-cgt", "--cg-tol", - "Relative CG tolerance (velocity linear solve)."); - args.AddOption(&cg_max_iter, "-cgm", "--cg-max-steps", - "Maximum number of CG iterations (velocity linear solve)."); - args.AddOption(&max_tsteps, "-ms", "--max-steps", - "Maximum number of steps (negative means no restriction)."); - args.AddOption(&p_assembly, "-pa", "--partial-assembly", "-fa", - "--full-assembly", - "Activate 1D tensor-based assembly (partial assembly)."); - - args.AddOption(&amr, "-amr", "--enable-amr", "-no-amr", "--disable-amr", - "Experimental adaptive mesh refinement (problem 1 only)."); - args.AddOption(&ref_threshold, "-rt", "--ref-threshold", - "AMR refinement threshold."); - args.AddOption(&deref_threshold, "-dt", "--deref-threshold", - "AMR derefinement threshold (0 = no derefinement)."); - - args.AddOption(&visualization, "-vis", "--visualization", "-no-vis", - "--no-visualization", - "Enable or disable GLVis visualization."); - args.AddOption(&vis_steps, "-vs", "--visualization-steps", - "Visualize every n-th timestep."); - args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit", - "Enable or disable VisIt visualization."); - args.AddOption(&gfprint, "-print", "--print", "-no-print", "--no-print", - "Enable or disable result output (files in mfem format)."); - args.AddOption(&basename, "-k", "--outputfilename", - "Name of the visit dump files"); - - args.AddOption(&partition_type, "-pt", "--partition", - "Customized x/y/z Cartesian MPI partitioning of the serial mesh.\n\t" - "Here x,y,z are relative task ratios in each direction.\n\t" - "Example: with 48 mpi tasks and -pt 321, one would get a Cartesian\n\t" - "partition of the serial mesh by (6,4,2) MPI tasks in (x,y,z).\n\t" - "NOTE: the serially refined mesh must have the appropriate number\n\t" - "of zones in each direction, e.g., the number of zones in direction x\n\t" - "must be divisible by the number of MPI tasks in direction x.\n\t" - "Available options: 11, 21, 111, 211, 221, 311, 321, 322, 432."); - args.Parse(); - if (!args.Good()) - { - if (mpi.Root()) { args.PrintUsage(cout); } - return 1; - } - - if (amr && problem != 1) - { - if (mpi.Root()) { cout << "AMR only supported for problem 1." << endl; } - return 0; - } - - if (mpi.Root()) { args.PrintOptions(cout); } - - // Read the serial mesh from the given mesh file on all processors. - // Refine the mesh in serial to increase the resolution. - Mesh *mesh = new Mesh(mesh_file, 1, 1); - const int dim = mesh->Dimension(); - if (!amr) - { - for (int lev = 0; lev < rs_levels; lev++) - { - mesh->UniformRefinement(); - } - } - else - { - // Initial refinement for AMR demo on Sedov - mesh->EnsureNCMesh(); - for (int lev = 0; lev < rs_levels; lev++) - { - mesh->RefineAtVertex(Vertex(blast_position[0], blast_position[1], - blast_position[2]), - blast_amr_size); - } - } - - if (p_assembly && dim == 1) - { - p_assembly = false; - if (mpi.Root()) - { - cout << "Laghos does not support PA in 1D. Switching to FA." << endl; - } - } - if (p_assembly && amr) - { - p_assembly = false; - if (mpi.Root()) - { - cout << "Laghos currently does not support PA for AMR. " - "Switching to FA." << endl; - } - } - - // Parallel partitioning of the mesh. - ParMesh *pmesh = NULL; - const int num_tasks = mpi.WorldSize(); int unit; - int *nxyz = new int[3]; - switch (partition_type) - { - case 11: - case 111: - unit = floor(pow(num_tasks, 1.0 / dim) + 1e-2); - for (int d = 0; d < dim; d++) { nxyz[d] = unit; } - if (dim == 2) { nxyz[2] = 0; } - break; - case 21: // 2D - unit = floor(pow(num_tasks / 2, 1.0 / 2) + 1e-2); - nxyz[0] = 2 * unit; nxyz[1] = unit; nxyz[2] = 0; - break; - case 211: // 3D. - unit = floor(pow(num_tasks / 2, 1.0 / 3) + 1e-2); - nxyz[0] = 2 * unit; nxyz[1] = unit; nxyz[2] = unit; - break; - case 221: // 3D. - unit = floor(pow(num_tasks / 4, 1.0 / 3) + 1e-2); - nxyz[0] = 2 * unit; nxyz[1] = 2 * unit; nxyz[2] = unit; - break; - case 311: // 3D. - unit = floor(pow(num_tasks / 3, 1.0 / 3) + 1e-2); - nxyz[0] = 3 * unit; nxyz[1] = unit; nxyz[2] = unit; - break; - case 321: // 3D. - unit = floor(pow(num_tasks / 6, 1.0 / 3) + 1e-2); - nxyz[0] = 3 * unit; nxyz[1] = 2 * unit; nxyz[2] = unit; - break; - case 322: // 3D. - unit = floor(pow(2 * num_tasks / 3, 1.0 / 3) + 1e-2); - nxyz[0] = 3 * unit / 2; nxyz[1] = unit; nxyz[2] = unit; - break; - case 432: // 3D. - unit = floor(pow(num_tasks / 3, 1.0 / 3) + 1e-2); - nxyz[0] = 2 * unit; nxyz[1] = 3 * unit / 2; nxyz[2] = unit; - break; - default: - if (myid == 0) - { - cout << "Unknown partition type: " << partition_type << '\n'; - } - delete mesh; - MPI_Finalize(); - return 3; - } - int product = 1; - for (int d = 0; d < dim; d++) { product *= nxyz[d]; } - if (myid == 0) - { - cout << nxyz[0] << " " << nxyz[1] << " " << nxyz[2] << " " - << product << " " << num_tasks << endl; - } - if (product == num_tasks) - { - int *partitioning = mesh->CartesianPartitioning(nxyz); - pmesh = new ParMesh(MPI_COMM_WORLD, *mesh, partitioning); - delete partitioning; - } - else - { - if (myid == 0) - { - cout << "Non-Cartesian partitioning through METIS will be used.\n"; -#ifndef MFEM_USE_METIS - cout << "MFEM was built without METIS. " - << "Adjust the number of tasks to use a Cartesian split." << endl; -#endif - } -#ifndef MFEM_USE_METIS - return 1; -#endif - pmesh = new ParMesh(MPI_COMM_WORLD, *mesh); - } - delete [] nxyz; - delete mesh; - - // Refine the mesh further in parallel to increase the resolution. - for (int lev = 0; lev < rp_levels; lev++) - { - pmesh->UniformRefinement(); - } - - int nzones = pmesh->GetNE(), nzones_min, nzones_max; - MPI_Reduce(&nzones, &nzones_min, 1, MPI_INT, MPI_MIN, 0, pmesh->GetComm()); - MPI_Reduce(&nzones, &nzones_max, 1, MPI_INT, MPI_MAX, 0, pmesh->GetComm()); - if (myid == 0) - { cout << "Zones min/max: " << nzones_min << " " << nzones_max << endl; } - - int amr_max_level = rs_levels + rp_levels; - - // Define the parallel finite element spaces. We use: - // - H1 (Gauss-Lobatto, continuous) for position and velocity. - // - L2 (Bernstein, discontinuous) for specific internal energy. - L2_FECollection L2FEC(order_e, dim/*, BasisType::Positive*/); - H1_FECollection H1FEC(order_v, dim); - ParFiniteElementSpace L2FESpace(pmesh, &L2FEC); - ParFiniteElementSpace H1FESpace(pmesh, &H1FEC, pmesh->Dimension()); - - // Boundary conditions: all tests use v.n = 0 on the boundary, and we assume - // that the boundaries are straight. - Array ess_tdofs; - int bdr_attr_max = pmesh->bdr_attributes.Max(); - GetZeroBCDofs(pmesh, &H1FESpace, bdr_attr_max, ess_tdofs); - - // Define the explicit ODE solver used for time integration. - ODESolver *ode_solver = NULL; - switch (ode_solver_type) - { - case 1: ode_solver = new ForwardEulerSolver; break; - case 2: ode_solver = new RK2Solver(0.5); break; - case 3: ode_solver = new RK3SSPSolver; break; - case 4: ode_solver = new RK4Solver; break; - case 6: ode_solver = new RK6Solver; break; - default: - if (myid == 0) - { - cout << "Unknown ODE solver type: " << ode_solver_type << '\n'; - } - delete pmesh; - MPI_Finalize(); - return 3; - } - - HYPRE_Int glob_size_l2 = L2FESpace.GlobalTrueVSize(); - HYPRE_Int glob_size_h1 = H1FESpace.GlobalTrueVSize(); - - if (mpi.Root()) - { - cout << "Number of kinematic (position, velocity) dofs: " - << glob_size_h1 << endl; - cout << "Number of specific internal energy dofs: " - << glob_size_l2 << endl; - } - - int Vsize_l2 = L2FESpace.GetVSize(); - int Vsize_h1 = H1FESpace.GetVSize(); - - // The monolithic BlockVector stores unknown fields as: - // - 0 -> position - // - 1 -> velocity - // - 2 -> specific internal energy - - Array true_offset(4); - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - BlockVector S(true_offset); - - // Define GridFunction objects for the position, velocity and specific - // internal energy. There is no function for the density, as we can always - // compute the density values given the current mesh position, using the - // property of pointwise mass conservation. - ParGridFunction x_gf, v_gf, e_gf; - x_gf.MakeRef(&H1FESpace, S, true_offset[0]); - v_gf.MakeRef(&H1FESpace, S, true_offset[1]); - e_gf.MakeRef(&L2FESpace, S, true_offset[2]); - - // Initialize x_gf using the starting mesh coordinates. This also links the - // mesh positions to the values in x_gf. - pmesh->SetNodalGridFunction(&x_gf); - - // Initialize the velocity. - VectorFunctionCoefficient v_coeff(pmesh->Dimension(), v0); - v_gf.ProjectCoefficient(v_coeff); - - // Initialize density and specific internal energy values. We interpolate in - // a non-positive basis to get the correct values at the dofs. Then we do an - // L2 projection to the positive basis in which we actually compute. The goal - // is to get a high-order representation of the initial condition. Note that - // this density is a temporary function and it will not be updated during the - // time evolution. - ParGridFunction rho0_gf(&L2FESpace); - { - FunctionCoefficient rho_coeff(hydrodynamics::rho0); - L2_FECollection l2_fec(order_e, pmesh->Dimension()); - ParFiniteElementSpace l2_fes(pmesh, &l2_fec); - - ParGridFunction l2_rho(&l2_fes), l2_e(&l2_fes); - l2_rho.ProjectCoefficient(rho_coeff); - rho0_gf.ProjectGridFunction(l2_rho); - - if (problem == 1) - { - // For the Sedov test, we use a delta function at the origin. - DeltaCoefficient e_coeff(blast_position[0], blast_position[1], - blast_position[2], blast_energy); - l2_e.ProjectCoefficient(e_coeff); - } - else - { - FunctionCoefficient e_coeff(e0); - l2_e.ProjectCoefficient(e_coeff); - } - e_gf.ProjectGridFunction(l2_e); - } - - // Space-dependent ideal gas coefficient over the Lagrangian mesh. - Coefficient *material_pcf = new FunctionCoefficient(hydrodynamics::gamma); - - // Additional details, depending on the problem. - int source = 0; bool visc = false; - switch (problem) - { - case 0: if (pmesh->Dimension() == 2) { source = 1; } - visc = false; break; - case 1: visc = true; break; - case 2: visc = true; break; - case 3: visc = true; break; - default: MFEM_ABORT("Wrong problem specification!"); - } - - LagrangianHydroOperator oper(S.Size(), H1FESpace, L2FESpace, - ess_tdofs, rho0_gf, source, cfl, material_pcf, - visc, p_assembly, cg_tol, cg_max_iter); - - if (amr) - { - // set a base for h0, this will be further divided in UpdateQuadratureData - // TODO: for AMR, the treatment of h0 needs more work - double elem_size = 0.5; // coarse element size (TODO calculate) - double h0 = elem_size / order_v; - oper.SetH0(h0); - } - - socketstream vis_rho, vis_v, vis_e; - char vishost[] = "localhost"; - int visport = 19916; - - ParGridFunction rho_gf; - if (visualization || visit) - { - oper.ComputeDensity(rho_gf); - } - - if (visualization) - { - // Make sure all MPI ranks have sent their 'v' solution before initiating - // another set of GLVis connections (one from each rank): - MPI_Barrier(pmesh->GetComm()); - - int Wx = 0, Wy = 0; // window position - const int Ww = 500, Wh = 500; // window size - int offx = Ww+10; // window offsets - - VisualizeField(vis_rho, vishost, visport, rho_gf, - "Density", Wx, Wy, Ww, Wh); - Wx += offx; - VisualizeField(vis_v, vishost, visport, v_gf, - "Velocity", Wx, Wy, Ww, Wh); - Wx += offx; - VisualizeField(vis_e, vishost, visport, e_gf, - "Specific Internal Energy", Wx, Wy, Ww, Wh); - } - - // Save data for VisIt visualization. - VisItDataCollection visit_dc(basename, pmesh); - if (visit) - { - visit_dc.RegisterField("Density", &rho_gf); - visit_dc.RegisterField("Velocity", &v_gf); - visit_dc.RegisterField("Specific Internal Energy", &e_gf); - visit_dc.SetCycle(0); - visit_dc.SetTime(0.0); - visit_dc.Save(); - } - - // Perform time-integration (looping over the time iterations, ti, with a - // time-step dt). The object oper is of type LagrangianHydroOperator that - // defines the Mult() method that used by the time integrators. - ode_solver->Init(oper); - oper.ResetTimeStepEstimate(); - double t = 0.0, dt = oper.GetTimeStepEstimate(S), t_old; - bool last_step = false; - int steps = 0; - BlockVector S_old(S); - for (int ti = 1; !last_step; ti++) - { - if (t + dt >= t_final) - { - dt = t_final - t; - last_step = true; - } - if (steps == max_tsteps) { last_step = true; } - - S_old = S; - t_old = t; - oper.ResetTimeStepEstimate(); - - // S is the vector of dofs, t is the current time, and dt is the time step - // to advance. - ode_solver->Step(S, t, dt); - steps++; - - // Adaptive time step control. - const double dt_est = oper.GetTimeStepEstimate(S); - if (dt_est < dt) - { - // Repeat (solve again) with a decreased time step - decrease of the - // time estimate suggests appearance of oscillations. - dt *= 0.85; - if (dt < numeric_limits::epsilon()) - { MFEM_ABORT("The time step crashed!"); } - t = t_old; - S = S_old; - oper.ResetQuadratureData(); - if (mpi.Root()) { cout << "Repeating step " << ti << endl; } - ti--; continue; - } - else if (dt_est > 1.25 * dt) { dt *= 1.02; } - - // Make sure that the mesh corresponds to the new solution state. - pmesh->NewNodes(x_gf, false); - - if (last_step || (ti % vis_steps) == 0) - { - double loc_norm = e_gf * e_gf, tot_norm; - MPI_Allreduce(&loc_norm, &tot_norm, 1, MPI_DOUBLE, MPI_SUM, - pmesh->GetComm()); - if (mpi.Root()) - { - cout << fixed; - cout << "step " << setw(5) << ti - << ",\tt = " << setw(5) << setprecision(4) << t - << ",\tdt = " << setw(5) << setprecision(6) << dt - << ",\t|e| = " << setprecision(10) - << sqrt(tot_norm) << endl; - } - - // Make sure all ranks have sent their 'v' solution before initiating - // another set of GLVis connections (one from each rank): - MPI_Barrier(pmesh->GetComm()); - - if (visualization || visit || gfprint) - { - oper.ComputeDensity(rho_gf); - } - if (visualization) - { - int Wx = 0, Wy = 0; // window position - int Ww = 500, Wh = 500; // window size - int offx = Ww+10; // window offsets - - VisualizeField(vis_rho, vishost, visport, rho_gf, - "Density", Wx, Wy, Ww, Wh); - Wx += offx; - VisualizeField(vis_v, vishost, visport, - v_gf, "Velocity", Wx, Wy, Ww, Wh); - Wx += offx; - VisualizeField(vis_e, vishost, visport, e_gf, - "Specific Internal Energy", Wx, Wy, Ww,Wh); - Wx += offx; - } - - if (visit) - { - visit_dc.SetCycle(ti); - visit_dc.SetTime(t); - visit_dc.Save(); - } - - if (gfprint) - { - ostringstream mesh_name, rho_name, v_name, e_name; - mesh_name << basename << "_" << ti - << "_mesh." << setfill('0') << setw(6) << myid; - rho_name << basename << "_" << ti - << "_rho." << setfill('0') << setw(6) << myid; - v_name << basename << "_" << ti - << "_v." << setfill('0') << setw(6) << myid; - e_name << basename << "_" << ti - << "_e." << setfill('0') << setw(6) << myid; - - ofstream mesh_ofs(mesh_name.str().c_str()); - mesh_ofs.precision(8); - pmesh->Print(mesh_ofs); - mesh_ofs.close(); - - ofstream rho_ofs(rho_name.str().c_str()); - rho_ofs.precision(8); - rho_gf.Save(rho_ofs); - rho_ofs.close(); - - ofstream v_ofs(v_name.str().c_str()); - v_ofs.precision(8); - v_gf.Save(v_ofs); - v_ofs.close(); - - ofstream e_ofs(e_name.str().c_str()); - e_ofs.precision(8); - e_gf.Save(e_ofs); - e_ofs.close(); - } - } - - if (amr) - { - Vector &error_est = oper.GetZoneMaxVisc(); - - Vector v_max, v_min; - GetPerElementMinMax(v_gf, v_min, v_max); - - bool mesh_changed = false; - - // make a list of elements to refine - Array refs; - for (int i = 0; i < pmesh->GetNE(); i++) - { - if (error_est(i) > ref_threshold - && pmesh->pncmesh->GetElementDepth(i) < amr_max_level - && (v_min(i) < 1e-3 || ti < 50) // only refine the still area - ) - { - refs.Append(i); - } - } - - int nref = pmesh->ReduceInt(refs.Size()); - if (nref) - { - pmesh->GeneralRefinement(refs, 1, nc_limit); - mesh_changed = true; - - if (myid == 0) - { - cout << "Refined " << nref << " elements." << endl; - } - } - else if (deref_threshold) - { - oper.ComputeDensity(rho_gf); - - Vector rho_max, rho_min; - GetPerElementMinMax(rho_gf, rho_min, rho_max); - - // simple derefinement based on zone maximum rho in post-shock region - double rho_max_max = rho_max.Size() ? rho_max.Max() : 0.0; - double threshold, loc_threshold = deref_threshold * rho_max_max; - MPI_Allreduce(&loc_threshold, &threshold, 1, MPI_DOUBLE, MPI_MAX, - pmesh->GetComm()); - - // make sure the blast point is never derefined - Array elements; - FindElementsWithVertex(pmesh, Vertex(blast_position[0], - blast_position[1], - blast_position[2]), - blast_amr_size, elements); - for (int i = 0; i < elements.Size(); i++) - { - int index = elements[i]; - if (index >= 0) { rho_max(index) = 1e10; } - } - - // also, only derefine where the mesh is in motion, i.e. after the shock - for (int i = 0; i < pmesh->GetNE(); i++) - { - if (v_min(i) < 0.1) { rho_max(i) = 1e10; } - } - - const int op = 2; // maximum value of fine elements - mesh_changed = pmesh->DerefineByError(rho_max, threshold, - nc_limit, op); - if (mesh_changed && myid == 0) - { - cout << "Derefined, threshold = " << threshold << endl; - } - } - - if (mesh_changed) - { - // update state and operator - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - oper.AMRUpdate(S, true); - - pmesh->Rebalance(); - - // update state and operator - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf); - oper.AMRUpdate(S, false); - - GetZeroBCDofs(pmesh, &H1FESpace, bdr_attr_max, ess_tdofs); - - ode_solver->Init(oper); - - //H1FESpace.PrintPartitionStats(); - } - } - } - - switch (ode_solver_type) - { - case 2: steps *= 2; break; - case 3: steps *= 3; break; - case 4: steps *= 4; break; - case 6: steps *= 6; - } - oper.PrintTimingData(mpi.Root(), steps); - - if (visualization) - { - vis_v.close(); - vis_e.close(); - } - - // Free the used memory. - delete ode_solver; - delete pmesh; - delete material_pcf; - - return 0; -} - - -void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace *pspace, - int bdr_attr_max, Array &ess_tdofs) -{ - ess_tdofs.SetSize(0); - Array ess_bdr(bdr_attr_max), tdofs1d; - for (int d = 0; d < pmesh->Dimension(); d++) - { - // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, i.e., we must - // enforce v_x/y/z = 0 for the velocity components. - ess_bdr = 0; ess_bdr[d] = 1; - pspace->GetEssentialTrueDofs(ess_bdr, tdofs1d, d); - ess_tdofs.Append(tdofs1d); - } -} - -void AMRUpdate(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf) -{ - ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); - ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - - H1FESpace->Update(); - L2FESpace->Update(); - - int Vsize_h1 = H1FESpace->GetVSize(); - int Vsize_l2 = L2FESpace->GetVSize(); - - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - - S_tmp = S; - S.Update(true_offset); - - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); - - x_gf.MakeRef(H1FESpace, S, true_offset[0]); - v_gf.MakeRef(H1FESpace, S, true_offset[1]); - e_gf.MakeRef(L2FESpace, S, true_offset[2]); - - S_tmp.Update(true_offset); -} - -void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements) -{ - Array v; - - for (int i = 0; i < mesh->GetNE(); i++) - { - mesh->GetElementVertices(i, v); - for (int j = 0; j < v.Size(); j++) - { - double dist = 0.0; - for (int l = 0; l < mesh->SpaceDimension(); l++) - { - double d = vert(l) - mesh->GetVertex(v[j])[l]; - dist += d*d; - } - if (dist <= size*size) { elements.Append(i); break; } - } - } -} - -void Pow(Vector &vec, double p) -{ - for (int i = 0; i < vec.Size(); i++) - { - vec(i) = std::pow(vec(i), p); - } -} - -void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order) -{ - const FiniteElementSpace *space = gf.FESpace(); - int ne = space->GetNE(); - - if (int_order < 0) { int_order = space->GetOrder(0) + 1; } - - elem_min.SetSize(ne); - elem_max.SetSize(ne); - - Vector vals, tmp; - for (int i = 0; i < ne; i++) - { - int geom = space->GetFE(i)->GetGeomType(); - const IntegrationRule &ir = IntRules.Get(geom, int_order); - - gf.GetValues(i, ir, vals); - - if (space->GetVDim() > 1) - { - Pow(vals, 2.0); - for (int vd = 1; vd < space->GetVDim(); vd++) - { - gf.GetValues(i, ir, tmp, vd+1); - Pow(tmp, 2.0); - vals += tmp; - } - Pow(vals, 0.5); - } - - elem_min(i) = vals.Min(); - elem_max(i) = vals.Max(); - } -} - -namespace mfem -{ - -namespace hydrodynamics -{ - -double rho0(const Vector &x) -{ - switch (problem) - { - case 0: return 1.0; - case 1: return 1.0; - case 2: if (x(0) < 0.5) { return 1.0; } - else { return 0.1; } - case 3: if (x(0) > 1.0 && x(1) <= 1.5) { return 1.0; } - else { return 0.125; } - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -double gamma(const Vector &x) -{ - switch (problem) - { - case 0: return 5./3.; - case 1: return 1.4; - case 2: return 1.4; - case 3: if (x(0) > 1.0 && x(1) <= 1.5) { return 1.4; } - else { return 1.5; } - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -void v0(const Vector &x, Vector &v) -{ - switch (problem) - { - case 0: - v(0) = sin(M_PI*x(0)) * cos(M_PI*x(1)); - v(1) = -cos(M_PI*x(0)) * sin(M_PI*x(1)); - if (x.Size() == 3) - { - v(0) *= cos(M_PI*x(2)); - v(1) *= cos(M_PI*x(2)); - v(2) = 0.0; - } - break; - case 1: v = 0.0; break; - case 2: v = 0.0; break; - case 3: v = 0.0; break; - default: MFEM_ABORT("Bad number given for problem id!"); - } -} - -double e0(const Vector &x) -{ - switch (problem) - { - case 0: - { - const double denom = 2.0 / 3.0; // (5/3 - 1) * density. - double val; - if (x.Size() == 2) - { - val = 1.0 + (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) / 4.0; - } - else - { - val = 100.0 + ((cos(2*M_PI*x(2)) + 2) * - (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) - 2) / 16.0; - } - return val/denom; - } - case 1: return 0.0; // This case is initialized in main(). - case 2: if (x(0) < 0.5) { return 1.0 / rho0(x) / (gamma(x) - 1.0); } - else { return 0.1 / rho0(x) / (gamma(x) - 1.0); } - case 3: if (x(0) > 1.0) { return 0.1 / rho0(x) / (gamma(x) - 1.0); } - else { return 1.0 / rho0(x) / (gamma(x) - 1.0); } - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -} // namespace hydrodynamics - -} // namespace mfem - -void display_banner(ostream & os) -{ - os << endl - << " __ __ " << endl - << " / / ____ ____ / /_ ____ _____ " << endl - << " / / / __ `/ __ `/ __ \\/ __ \\/ ___/ " << endl - << " / /___/ /_/ / /_/ / / / / /_/ (__ ) " << endl - << " /_____/\\__,_/\\__, /_/ /_/\\____/____/ " << endl - << " /____/ " << endl << endl; -} diff --git a/amr/laghos_assembly.cpp b/amr/laghos_assembly.cpp deleted file mode 100644 index 6888ba46..00000000 --- a/amr/laghos_assembly.cpp +++ /dev/null @@ -1,1072 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project (17-SC-20-SC) -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. - -#include "laghos_assembly.hpp" - -#ifdef MFEM_USE_MPI - -using namespace std; - -namespace mfem -{ - -namespace hydrodynamics -{ - -const Tensors1D *tensors1D = NULL; -const FastEvaluator *evaluator = NULL; - -Tensors1D::Tensors1D(int H1order, int L2order, int nqp1D) - : HQshape1D(H1order + 1, nqp1D), - HQgrad1D(H1order + 1, nqp1D), - LQshape1D(L2order + 1, nqp1D) -{ - // In this miniapp we assume: - // - Gauss-Legendre quadrature points. - // - Gauss-Lobatto continuous kinematic basis. - // - Bernstein discontinuous thermodynamic basis. - - const double *quad1D_pos = poly1d.GetPoints(nqp1D - 1, - Quadrature1D::GaussLegendre); - Poly_1D::Basis &basisH1 = poly1d.GetBasis(H1order, - Quadrature1D::GaussLobatto); - Vector col, grad_col; - for (int q = 0; q < nqp1D; q++) - { - HQshape1D.GetColumnReference(q, col); - HQgrad1D.GetColumnReference(q, grad_col); - basisH1.Eval(quad1D_pos[q], col, grad_col); - } - for (int q = 0; q < nqp1D; q++) - { - LQshape1D.GetColumnReference(q, col); - poly1d.CalcBernstein(L2order, quad1D_pos[q], col); - } -} - -void FastEvaluator::GetL2Values(const Vector &vecL2, Vector &vecQ) const -{ - const int nL2dof1D = tensors1D->LQshape1D.Height(), - nqp1D = tensors1D->LQshape1D.Width(); - if (dim == 2) - { - DenseMatrix E(vecL2.GetData(), nL2dof1D, nL2dof1D); - DenseMatrix LQ(nL2dof1D, nqp1D); - - vecQ.SetSize(nqp1D * nqp1D); - DenseMatrix QQ(vecQ.GetData(), nqp1D, nqp1D); - - // LQ_j2_k1 = E_j1_j2 LQs_j1_k1 -- contract in x direction. - // QQ_k1_k2 = LQ_j2_k1 LQs_j2_k2 -- contract in y direction. - MultAtB(E, tensors1D->LQshape1D, LQ); - MultAtB(LQ, tensors1D->LQshape1D, QQ); - } - else - { - DenseMatrix E(vecL2.GetData(), nL2dof1D*nL2dof1D, nL2dof1D); - DenseMatrix LL_Q(nL2dof1D * nL2dof1D, nqp1D), - L_LQ(LL_Q.GetData(), nL2dof1D, nL2dof1D*nqp1D), - Q_LQ(nqp1D, nL2dof1D*nqp1D); - - vecQ.SetSize(nqp1D * nqp1D * nqp1D); - DenseMatrix QQ_Q(vecQ.GetData(), nqp1D * nqp1D, nqp1D); - - // LLQ_j1_j2_k3 = E_j1_j2_j3 LQs_j3_k3 -- contract in z direction. - // QLQ_k1_j2_k3 = LQs_j1_k1 LLQ_j1_j2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QLQ_k1_j2_k3 LQs_j2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(E, tensors1D->LQshape1D, LL_Q); - MultAtB(tensors1D->LQshape1D, L_LQ, Q_LQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int j2 = 0; j2 < nL2dof1D; j2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += - Q_LQ(k1, j2 + k3*nL2dof1D) * tensors1D->LQshape1D(j2, k2); - } - } - } - } - } -} - -void FastEvaluator::GetVectorGrad(const DenseMatrix &vec, DenseTensor &J) const -{ - const int nH1dof1D = tensors1D->HQshape1D.Height(), - nqp1D = tensors1D->LQshape1D.Width(); - DenseMatrix X; - - if (dim == 2) - { - const int nH1dof = nH1dof1D * nH1dof1D; - DenseMatrix HQ(nH1dof1D, nqp1D), QQ(nqp1D, nqp1D); - Vector x(nH1dof); - - const H1_QuadrilateralElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - for (int c = 0; c < 2; c++) - { - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - for (int j = 0; j < nH1dof; j++) { x[j] = vec(dof_map[j], c); } - X.UseExternalData(x.GetData(), nH1dof1D, nH1dof1D); - - // HQ_i2_k1 = X_i1_i2 HQg_i1_k1 -- gradients in x direction. - // QQ_k1_k2 = HQ_i2_k1 HQs_i2_k2 -- contract in y direction. - MultAtB(X, tensors1D->HQgrad1D, HQ); - MultAtB(HQ, tensors1D->HQshape1D, QQ); - - // Set the (c,0) component of the Jacobians at all quadrature points. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - const int idx = k2 * nqp1D + k1; - J(idx)(c, 0) = QQ(k1, k2); - } - } - - // HQ_i2_k1 = X_i1_i2 HQs_i1_k1 -- contract in x direction. - // QQ_k1_k2 = HQ_i2_k1 HQg_i2_k2 -- gradients in y direction. - MultAtB(X, tensors1D->HQshape1D, HQ); - MultAtB(HQ, tensors1D->HQgrad1D, QQ); - - // Set the (c,1) component of the Jacobians at all quadrature points. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - const int idx = k2 * nqp1D + k1; - J(idx)(c, 1) = QQ(k1, k2); - } - } - } - } - else - { - const int nH1dof = nH1dof1D * nH1dof1D * nH1dof1D; - DenseMatrix HH_Q(nH1dof1D * nH1dof1D, nqp1D), - H_HQ(HH_Q.GetData(), nH1dof1D, nH1dof1D * nqp1D), - Q_HQ(nqp1D, nH1dof1D*nqp1D), QQ_Q(nqp1D * nqp1D, nqp1D); - Vector x(nH1dof); - - const H1_HexahedronElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - for (int c = 0; c < 3; c++) - { - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - for (int j = 0; j < nH1dof; j++) { x[j] = vec(dof_map[j], c); } - X.UseExternalData(x.GetData(), nH1dof1D * nH1dof1D, nH1dof1D); - - // HHQ_i1_i2_k3 = X_i1_i2_i3 HQs_i3_k3 -- contract in z direction. - // QHQ_k1_i2_k3 = HQg_i1_k1 HHQ_i1_i2_k3 -- gradients in x direction. - // QQQ_k1_k2_k3 = QHQ_k1_i2_k3 HQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(X, tensors1D->HQshape1D, HH_Q); - MultAtB(tensors1D->HQgrad1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQshape1D(i2, k2); - } - } - } - } - // Set the (c,0) component of the Jacobians at all quadrature points. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - const int idx = k3*nqp1D*nqp1D + k2*nqp1D + k1; - J(idx)(c, 0) = QQ_Q(k1 + k2*nqp1D, k3); - } - } - } - - // HHQ_i1_i2_k3 = X_i1_i2_i3 HQs_i3_k3 -- contract in z direction. - // QHQ_k1_i2_k3 = HQs_i1_k1 HHQ_i1_i2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QHQ_k1_i2_k3 HQg_i2_k2 -- gradients in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(X, tensors1D->HQshape1D, HH_Q); - MultAtB(tensors1D->HQshape1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQgrad1D(i2, k2); - } - } - } - } - // Set the (c,1) component of the Jacobians at all quadrature points. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - const int idx = k3*nqp1D*nqp1D + k2*nqp1D + k1; - J(idx)(c, 1) = QQ_Q(k1 + k2*nqp1D, k3); - } - } - } - - // HHQ_i1_i2_k3 = X_i1_i2_i3 HQg_i3_k3 -- gradients in z direction. - // QHQ_k1_i2_k3 = HQs_i1_k1 HHQ_i1_i2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QHQ_k1_i2_k3 HQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(X, tensors1D->HQgrad1D, HH_Q); - MultAtB(tensors1D->HQshape1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQshape1D(i2, k2); - } - } - } - } - // Set the (c,2) component of the Jacobians at all quadrature points. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - const int idx = k3*nqp1D*nqp1D + k2*nqp1D + k1; - J(idx)(c, 2) = QQ_Q(k1 + k2*nqp1D, k3); - } - } - } - } - } -} - -void DensityIntegrator::AssembleRHSElementVect(const FiniteElement &fe, - ElementTransformation &Tr, - Vector &elvect) -{ - const int ip_cnt = IntRule->GetNPoints(); - Vector shape(fe.GetDof()); - - elvect.SetSize(fe.GetDof()); - elvect = 0.0; - - for (int q = 0; q < ip_cnt; q++) - { - fe.CalcShape(IntRule->IntPoint(q), shape); - // Note that rhoDetJ = rho0DetJ0. - shape *= quad_data.rho0DetJ0w(Tr.ElementNo*ip_cnt + q); - elvect += shape; - } -} - -void ForceIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe, - const FiniteElement &test_fe, - ElementTransformation &Trans, - DenseMatrix &elmat) -{ - const int nqp = IntRule->GetNPoints(); - const int dim = trial_fe.GetDim(); - const int zone_id = Trans.ElementNo; - const int h1dofs_cnt = test_fe.GetDof(); - const int l2dofs_cnt = trial_fe.GetDof(); - - elmat.SetSize(h1dofs_cnt*dim, l2dofs_cnt); - elmat = 0.0; - - DenseMatrix vshape(h1dofs_cnt, dim), loc_force(h1dofs_cnt, dim); - Vector shape(l2dofs_cnt), Vloc_force(loc_force.Data(), h1dofs_cnt*dim); - - for (int q = 0; q < nqp; q++) - { - const IntegrationPoint &ip = IntRule->IntPoint(q); - - // Form stress:grad_shape at the current point. - test_fe.CalcDShape(ip, vshape); - for (int i = 0; i < h1dofs_cnt; i++) - { - for (int vd = 0; vd < dim; vd++) // Velocity components. - { - loc_force(i, vd) = 0.0; - for (int gd = 0; gd < dim; gd++) // Gradient components. - { - loc_force(i, vd) += - quad_data.stressJinvT(vd)(zone_id*nqp + q, gd) * vshape(i,gd); - } - } - } - - trial_fe.CalcShape(ip, shape); - AddMultVWt(Vloc_force, shape, elmat); - } -} - -void ForcePAOperator::Mult(const Vector &vecL2, Vector &vecH1) const -{ - if (dim == 2) { MultQuad(vecL2, vecH1); } - else if (dim == 3) { MultHex(vecL2, vecH1); } - else { MFEM_ABORT("Unsupported dimension"); } -} - -void ForcePAOperator::MultTranspose(const Vector &vecH1, Vector &vecL2) const -{ - if (dim == 2) { MultTransposeQuad(vecH1, vecL2); } - else if (dim == 3) { MultTransposeHex(vecH1, vecL2); } - else { MFEM_ABORT("Unsupported dimension"); } -} - -// Force matrix action on quadrilateral elements in 2D. -void ForcePAOperator::MultQuad(const Vector &vecL2, Vector &vecH1) const -{ - const int nH1dof1D = tensors1D->HQshape1D.Height(), - nL2dof1D = tensors1D->LQshape1D.Height(), - nqp1D = tensors1D->HQshape1D.Width(), - nqp = nqp1D * nqp1D, - nH1dof = nH1dof1D * nH1dof1D; - Array h1dofs, l2dofs; - Vector e(nL2dof1D * nL2dof1D); - DenseMatrix E(e.GetData(), nL2dof1D, nL2dof1D); - DenseMatrix LQ(nL2dof1D, nqp1D), HQ(nH1dof1D, nqp1D), QQ(nqp1D, nqp1D), - HHx(nH1dof1D, nH1dof1D), HHy(nH1dof1D, nH1dof1D); - // Quadrature data for a specific direction. - DenseMatrix QQd(nqp1D, nqp1D); - double *data_qd = QQd.GetData(), *data_q = QQ.GetData(); - - const H1_QuadrilateralElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - vecH1 = 0.0; - for (int z = 0; z < nzones; z++) - { - // Note that the local numbering for L2 is the tensor numbering. - L2FESpace.GetElementDofs(z, l2dofs); - vecL2.GetSubVector(l2dofs, e); - - // LQ_j2_k1 = E_j1_j2 LQs_j1_k1 -- contract in x direction. - // QQ_k1_k2 = LQ_j2_k1 LQs_j2_k2 -- contract in y direction. - MultAtB(E, tensors1D->LQshape1D, LQ); - MultAtB(LQ, tensors1D->LQshape1D, QQ); - - // Iterate over the components (x and y) of the result. - for (int c = 0; c < 2; c++) - { - // QQd_k1_k2 *= stress_k1_k2(c,0) -- stress that scales d[v_c]_dx. - // HQ_i2_k1 = HQs_i2_k2 QQ_k1_k2 -- contract in y direction. - // HHx_i1_i2 = HQg_i1_k1 HQ_i2_k1 -- gradients in x direction. - double *d = quad_data->stressJinvT(c).GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { data_qd[q] = data_q[q] * d[q]; }; - MultABt(tensors1D->HQshape1D, QQd, HQ); - MultABt(tensors1D->HQgrad1D, HQ, HHx); - - // QQd_k1_k2 *= stress_k1_k2(c,1) -- stress that scales d[v_c]_dy. - // HQ_i2_k1 = HQg_i2_k2 QQ_k1_k2 -- gradients in y direction. - // HHy_i1_i2 = HQ_i1_k1 HQ_i2_k1 -- contract in x direction. - d = quad_data->stressJinvT(c).GetData() + 1*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { data_qd[q] = data_q[q] * d[q]; }; - MultABt(tensors1D->HQgrad1D, QQd, HQ); - MultABt(tensors1D->HQshape1D, HQ, HHy); - - // Set the c-component of the result. - H1FESpace.GetElementVDofs(z, h1dofs); - for (int i1 = 0; i1 < nH1dof1D; i1++) - { - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - // Transfer from the mfem's H1 local numbering to the tensor - // structure numbering. - const int idx = i2 * nH1dof1D + i1; - vecH1[h1dofs[c*nH1dof + dof_map[idx]]] += - HHx(i1, i2) + HHy(i1, i2); - } - } - } - } -} - -// Force matrix action on hexahedral elements in 3D. -void ForcePAOperator::MultHex(const Vector &vecL2, Vector &vecH1) const -{ - const int nH1dof1D = tensors1D->HQshape1D.Height(), - nL2dof1D = tensors1D->LQshape1D.Height(), - nqp1D = tensors1D->HQshape1D.Width(), - nqp = nqp1D * nqp1D * nqp1D, - nH1dof = nH1dof1D * nH1dof1D * nH1dof1D; - Array h1dofs, l2dofs; - - Vector e(nL2dof1D * nL2dof1D * nL2dof1D); - DenseMatrix E(e.GetData(), nL2dof1D*nL2dof1D, nL2dof1D); - - DenseMatrix HH_Q(nH1dof1D * nH1dof1D, nqp1D), - H_HQ(HH_Q.GetData(), nH1dof1D, nH1dof1D*nqp1D), - Q_HQ(nqp1D, nH1dof1D*nqp1D); - DenseMatrix LL_Q(nL2dof1D * nL2dof1D, nqp1D), - L_LQ(LL_Q.GetData(), nL2dof1D, nL2dof1D*nqp1D), - Q_LQ(nqp1D, nL2dof1D*nqp1D); - DenseMatrix QQ_Q(nqp1D * nqp1D, nqp1D), QQ_Qc(nqp1D * nqp1D, nqp1D); - double *qqq = QQ_Q.GetData(), *qqqc = QQ_Qc.GetData(); - DenseMatrix HHHx(nH1dof1D * nH1dof1D, nH1dof1D), - HHHy(nH1dof1D * nH1dof1D, nH1dof1D), - HHHz(nH1dof1D * nH1dof1D, nH1dof1D); - - const H1_HexahedronElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - vecH1 = 0.0; - for (int z = 0; z < nzones; z++) - { - // Note that the local numbering for L2 is the tensor numbering. - L2FESpace.GetElementDofs(z, l2dofs); - vecL2.GetSubVector(l2dofs, e); - - // LLQ_j1_j2_k3 = E_j1_j2_j3 LQs_j3_k3 -- contract in z direction. - // QLQ_k1_j2_k3 = LQs_j1_k1 LLQ_j1_j2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QLQ_k1_j2_k3 LQs_j2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(E, tensors1D->LQshape1D, LL_Q); - MultAtB(tensors1D->LQshape1D, L_LQ, Q_LQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int j2 = 0; j2 < nL2dof1D; j2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += - Q_LQ(k1, j2 + k3*nL2dof1D) * tensors1D->LQshape1D(j2, k2); - } - } - } - } - - // Iterate over the components (x, y, z) of the result. - for (int c = 0; c < 3; c++) - { - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,0) -- stress scaling d[v_c]_dx. - double *d = quad_data->stressJinvT(c).GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] = qqq[q] * d[q]; }; - - // QHQ_k1_i2_k3 = QQQc_k1_k2_k3 HQs_i2_k2 -- contract in y direction. - // The first step does some reordering (it's not product of matrices). - // HHQ_i1_i2_k3 = HQg_i1_k1 QHQ_k1_i2_k3 -- gradients in x direction. - // HHHx_i1_i2_i3 = HHQ_i1_i2_k3 HQs_i3_k3 -- contract in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) += - QQ_Qc(k1 + nqp1D*k2, k3) * tensors1D->HQshape1D(i2, k2); - } - } - } - } - mfem::Mult(tensors1D->HQgrad1D, Q_HQ, H_HQ); - MultABt(HH_Q, tensors1D->HQshape1D, HHHx); - - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,1) -- stress scaling d[v_c]_dy. - d = quad_data->stressJinvT(c).GetData() + 1*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] = qqq[q] * d[q]; }; - - // QHQ_k1_i2_k3 = QQQc_k1_k2_k3 HQg_i2_k2 -- gradients in y direction. - // The first step does some reordering (it's not product of matrices). - // HHQ_i1_i2_k3 = HQs_i1_k1 QHQ_k1_i2_k3 -- contract in x direction. - // HHHy_i1_i2_i3 = HHQ_i1_i2_k3 HQs_i3_k3 -- contract in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) += - QQ_Qc(k1 + nqp1D*k2, k3) * tensors1D->HQgrad1D(i2, k2); - } - } - } - } - mfem::Mult(tensors1D->HQshape1D, Q_HQ, H_HQ); - MultABt(HH_Q, tensors1D->HQshape1D, HHHy); - - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,2) -- stress scaling d[v_c]_dz. - d = quad_data->stressJinvT(c).GetData() + 2*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] = qqq[q] * d[q]; }; - - // QHQ_k1_i2_k3 = QQQc_k1_k2_k3 HQg_i2_k2 -- contract in y direction. - // The first step does some reordering (it's not product of matrices). - // HHQ_i1_i2_k3 = HQs_i1_k1 QHQ_k1_i2_k3 -- contract in x direction. - // HHHz_i1_i2_i3 = HHQ_i1_i2_k3 HQs_i3_k3 -- gradients in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_HQ(k1, i2 + nH1dof1D*k3) += - QQ_Qc(k1 + nqp1D*k2, k3) * tensors1D->HQshape1D(i2, k2); - } - } - } - } - mfem::Mult(tensors1D->HQshape1D, Q_HQ, H_HQ); - MultABt(HH_Q, tensors1D->HQgrad1D, HHHz); - - // Set the c-component of the result. - H1FESpace.GetElementVDofs(z, h1dofs); - for (int i1 = 0; i1 < nH1dof1D; i1++) - { - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - for (int i3 = 0; i3 < nH1dof1D; i3++) - { - // Transfer from the mfem's H1 local numbering to the tensor - // structure numbering. - const int idx = i3*nH1dof1D*nH1dof1D + i2*nH1dof1D + i1; - vecH1[h1dofs[c*nH1dof + dof_map[idx]]] += - HHHx(i1 + i2*nH1dof1D, i3) + - HHHy(i1 + i2*nH1dof1D, i3) + - HHHz(i1 + i2*nH1dof1D, i3); - } - } - } - } - } -} - -// Transpose force matrix action on quadrilateral elements in 2D. -void ForcePAOperator::MultTransposeQuad(const Vector &vecH1, - Vector &vecL2) const -{ - const int nH1dof1D = tensors1D->HQshape1D.Height(), - nL2dof1D = tensors1D->LQshape1D.Height(), - nqp1D = tensors1D->HQshape1D.Width(), - nqp = nqp1D * nqp1D, - nH1dof = nH1dof1D * nH1dof1D; - Array h1dofs, l2dofs; - Vector v(nH1dof * 2), e(nL2dof1D * nL2dof1D); - DenseMatrix V, E(e.GetData(), nL2dof1D, nL2dof1D); - DenseMatrix HQ(nH1dof1D, nqp1D), LQ(nL2dof1D, nqp1D), - QQc(nqp1D, nqp1D), QQ(nqp1D, nqp1D); - double *qqc = QQc.GetData(); - - const H1_QuadrilateralElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - for (int z = 0; z < nzones; z++) - { - H1FESpace.GetElementVDofs(z, h1dofs); - - // Form (stress:grad_v) at all quadrature points. - QQ = 0.0; - for (int c = 0; c < 2; c++) - { - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - for (int j = 0; j < nH1dof; j++) - { - v[c*nH1dof + j] = vecH1[h1dofs[c*nH1dof + dof_map[j]]]; - } - // Connect to [v_c], i.e., the c-component of v. - V.UseExternalData(v.GetData() + c*nH1dof, nH1dof1D, nH1dof1D); - - // HQ_i2_k1 = V_i1_i2 HQg_i1_k1 -- gradients in x direction. - // QQc_k1_k2 = HQ_i2_k1 HQs_i2_k2 -- contract in y direction. - // QQc_k1_k2 *= stress_k1_k2(c,0) -- stress that scales d[v_c]_dx. - MultAtB(V, tensors1D->HQgrad1D, HQ); - MultAtB(HQ, tensors1D->HQshape1D, QQc); - double *d = quad_data->stressJinvT(c).GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { qqc[q] *= d[q]; } - // Add the (stress(c,0) * d[v_c]_dx) part of (stress:grad_v). - QQ += QQc; - - // HQ_i2_k1 = V_i1_i2 HQs_i1_k1 -- contract in x direction. - // QQc_k1_k2 = HQ_i2_k1 HQg_i2_k2 -- gradients in y direction. - // QQc_k1_k2 *= stress_k1_k2(c,1) -- stress that scales d[v_c]_dy. - MultAtB(V, tensors1D->HQshape1D, HQ); - MultAtB(HQ, tensors1D->HQgrad1D, QQc); - d = quad_data->stressJinvT(c).GetData() + 1*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { qqc[q] *= d[q]; } - // Add the (stress(c,1) * d[v_c]_dy) part of (stress:grad_v). - QQ += QQc; - } - - // LQ_j1_k2 = LQs_j1_k1 QQ_k1_k2 -- contract in x direction. - // E_j1_j2 = LQ_j1_k2 LQs_j2_k2 -- contract in y direction. - mfem::Mult(tensors1D->LQshape1D, QQ, LQ); - MultABt(LQ, tensors1D->LQshape1D, E); - - L2FESpace.GetElementDofs(z, l2dofs); - vecL2.SetSubVector(l2dofs, e); - } -} - -// Transpose force matrix action on hexahedral elements in 3D. -void ForcePAOperator::MultTransposeHex(const Vector &vecH1, Vector &vecL2) const -{ - const int nH1dof1D = tensors1D->HQshape1D.Height(), - nL2dof1D = tensors1D->LQshape1D.Height(), - nqp1D = tensors1D->HQshape1D.Width(), - nqp = nqp1D * nqp1D * nqp1D, - nH1dof = nH1dof1D * nH1dof1D * nH1dof1D; - Array h1dofs, l2dofs; - - Vector v(nH1dof * 3), e(nL2dof1D * nL2dof1D * nL2dof1D); - DenseMatrix V, E(e.GetData(), nL2dof1D * nL2dof1D, nL2dof1D); - - DenseMatrix HH_Q(nH1dof1D * nH1dof1D, nqp1D), - H_HQ(HH_Q.GetData(), nH1dof1D, nH1dof1D * nqp1D), - Q_HQ(nqp1D, nH1dof1D*nqp1D); - DenseMatrix LL_Q(nL2dof1D * nL2dof1D, nqp1D), - L_LQ(LL_Q.GetData(), nL2dof1D, nL2dof1D * nqp1D), - Q_LQ(nqp1D, nL2dof1D*nqp1D); - DenseMatrix QQ_Q(nqp1D * nqp1D, nqp1D), QQ_Qc(nqp1D * nqp1D, nqp1D); - double *qqqc = QQ_Qc.GetData(); - - const H1_HexahedronElement *fe = - dynamic_cast(H1FESpace.GetFE(0)); - const Array &dof_map = fe->GetDofMap(); - - for (int z = 0; z < nzones; z++) - { - H1FESpace.GetElementVDofs(z, h1dofs); - - // Form (stress:grad_v) at all quadrature points. - QQ_Q = 0.0; - for (int c = 0; c < 3; c++) - { - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - for (int j = 0; j < nH1dof; j++) - { - v[c*nH1dof + j] = vecH1[h1dofs[c*nH1dof + dof_map[j]]]; - } - // Connect to [v_c], i.e., the c-component of v. - V.UseExternalData(v.GetData() + c*nH1dof, nH1dof1D*nH1dof1D, nH1dof1D); - - // HHQ_i1_i2_k3 = V_i1_i2_i3 HQs_i3_k3 -- contract in z direction. - // QHQ_k1_i2_k3 = HQg_i1_k1 HHQ_i1_i2_k3 -- gradients in x direction. - // QQQc_k1_k2_k3 = QHQ_k1_i2_k3 HQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(V, tensors1D->HQshape1D, HH_Q); - MultAtB(tensors1D->HQgrad1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Qc(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Qc(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQshape1D(i2, k2); - } - } - } - } - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,0) -- stress scaling d[v_c]_dx. - double *d = quad_data->stressJinvT(c).GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] *= d[q]; }; - // Add the (stress(c,0) * d[v_c]_dx) part of (stress:grad_v). - QQ_Q += QQ_Qc; - - // HHQ_i1_i2_k3 = V_i1_i2_i3 HQs_i3_k3 -- contract in z direction. - // QHQ_k1_i2_k3 = HQs_i1_k1 HHQ_i1_i2_k3 -- contract in x direction. - // QQQc_k1_k2_k3 = QHQ_k1_i2_k3 HQg_i2_k2 -- gradients in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(V, tensors1D->HQshape1D, HH_Q); - MultAtB(tensors1D->HQshape1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Qc(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Qc(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQgrad1D(i2, k2); - } - } - } - } - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,1) -- stress scaling d[v_c]_dy. - d = quad_data->stressJinvT(c).GetData() + 1*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] *= d[q]; }; - // Add the (stress(c,1) * d[v_c]_dy) part of (stress:grad_v). - QQ_Q += QQ_Qc; - - // HHQ_i1_i2_k3 = V_i1_i2_i3 HQg_i3_k3 -- gradients in z direction. - // QHQ_k1_i2_k3 = HQs_i1_k1 HHQ_i1_i2_k3 -- contract in x direction. - // QQQc_k1_k2_k3 = QHQ_k1_i2_k3 HQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(V, tensors1D->HQgrad1D, HH_Q); - MultAtB(tensors1D->HQshape1D, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Qc(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < nH1dof1D; i2++) - { - QQ_Qc(k1 + nqp1D*k2, k3) += Q_HQ(k1, i2 + k3*nH1dof1D) * - tensors1D->HQshape1D(i2, k2); - } - } - } - } - // QQQc_k1_k2_k3 *= stress_k1_k2_k3(c,2) -- stress scaling d[v_c]_dz. - d = quad_data->stressJinvT(c).GetData() + 2*nzones*nqp + z*nqp; - for (int q = 0; q < nqp; q++) { qqqc[q] *= d[q]; }; - // Add the (stress(c,2) * d[v_c]_dz) part of (stress:grad_v). - QQ_Q += QQ_Qc; - } - - // QLQ_k1_j2_k3 = QQQ_k1_k2_k3 LQs_j2_k2 -- contract in y direction. - // The first step does some reordering (it's not product of matrices). - // LLQ_j1_j2_k3 = LQs_j1_k1 QLQ_k1_j2_k3 -- contract in x direction. - // E_j1_j2_i3 = LLQ_j1_j2_k3 LQs_j3_k3 -- contract in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int j2 = 0; j2 < nL2dof1D; j2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_LQ(k1, j2 + nL2dof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_LQ(k1, j2 + nL2dof1D*k3) += - QQ_Q(k1 + nqp1D*k2, k3) * tensors1D->LQshape1D(j2, k2); - } - } - } - } - mfem::Mult(tensors1D->LQshape1D, Q_LQ, L_LQ); - MultABt(LL_Q, tensors1D->LQshape1D, E); - - L2FESpace.GetElementDofs(z, l2dofs); - vecL2.SetSubVector(l2dofs, e); - } -} - -void MassPAOperator::Mult(const Vector &x, Vector &y) const -{ - const int comp_size = FESpace.GetNDofs(); - for (int c = 0; c < dim; c++) - { - Vector x_comp(x.GetData() + c * comp_size, comp_size), - y_comp(y.GetData() + c * comp_size, comp_size); - if (dim == 2) { MultQuad(x_comp, y_comp); } - else if (dim == 3) { MultHex(x_comp, y_comp); } - else { MFEM_ABORT("Unsupported dimension"); } - } -} - -// Mass matrix action on quadrilateral elements in 2D. -void MassPAOperator::MultQuad(const Vector &x, Vector &y) const -{ - const H1_QuadrilateralElement *fe_H1 = - dynamic_cast(FESpace.GetFE(0)); - const DenseMatrix &HQs = tensors1D->HQshape1D; - - const int ndof1D = HQs.Height(), nqp1D = HQs.Width(); - DenseMatrix HQ(ndof1D, nqp1D), QQ(nqp1D, nqp1D); - Vector xz(ndof1D * ndof1D), yz(ndof1D * ndof1D); - DenseMatrix X(xz.GetData(), ndof1D, ndof1D), - Y(yz.GetData(), ndof1D, ndof1D); - Array dofs; - double *qq = QQ.GetData(); - const int nqp = nqp1D * nqp1D; - - y.SetSize(x.Size()); - y = 0.0; - - for (int z = 0; z < nzones; z++) - { - FESpace.GetElementDofs(z, dofs); - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - const Array &dof_map = fe_H1->GetDofMap(); - for (int j = 0; j < xz.Size(); j++) - { - xz[j] = x[dofs[dof_map[j]]]; - } - - // HQ_i1_k2 = X_i1_i2 HQs_i2_k2 -- contract in y direction. - // QQ_k1_k2 = HQs_i1_k1 HQ_i1_k2 -- contract in x direction. - mfem::Mult(X, HQs, HQ); - MultAtB(HQs, HQ, QQ); - - // QQ_k1_k2 *= quad_data_k1_k2 -- scaling with quadrature values. - double *d = quad_data->rho0DetJ0w.GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { qq[q] *= d[q]; } - - // HQ_i1_k2 = HQs_i1_k1 QQ_k1_k2 -- contract in x direction. - // Y_i1_i2 = HQ_i1_k2 HQs_i2_k2 -- contract in y direction. - mfem::Mult(HQs, QQ, HQ); - MultABt(HQ, HQs, Y); - - for (int j = 0; j < yz.Size(); j++) - { - y[dofs[dof_map[j]]] += yz[j]; - } - } -} - -// Mass matrix action on hexahedral elements in 3D. -void MassPAOperator::MultHex(const Vector &x, Vector &y) const -{ - const H1_HexahedronElement *fe_H1 = - dynamic_cast(FESpace.GetFE(0)); - const DenseMatrix &HQs = tensors1D->HQshape1D; - - const int ndof1D = HQs.Height(), nqp1D = HQs.Width(); - DenseMatrix HH_Q(ndof1D * ndof1D, nqp1D); - DenseMatrix H_HQ(HH_Q.GetData(), ndof1D, ndof1D*nqp1D); - DenseMatrix Q_HQ(nqp1D, ndof1D*nqp1D); - DenseMatrix QQ_Q(nqp1D*nqp1D, nqp1D); - double *qqq = QQ_Q.GetData(); - Vector xz(ndof1D * ndof1D * ndof1D), yz(ndof1D * ndof1D * ndof1D); - DenseMatrix X(xz.GetData(), ndof1D*ndof1D, ndof1D), - Y(yz.GetData(), ndof1D*ndof1D, ndof1D); - const int nqp = nqp1D * nqp1D * nqp1D; - Array dofs; - - y.SetSize(x.Size()); - y = 0.0; - - for (int z = 0; z < nzones; z++) - { - FESpace.GetElementDofs(z, dofs); - // Transfer from the mfem's H1 local numbering to the tensor structure - // numbering. - const Array &dof_map = fe_H1->GetDofMap(); - for (int j = 0; j < xz.Size(); j++) - { - xz[j] = x[dofs[dof_map[j]]]; - } - - // HHQ_i1_i2_k3 = X_i1_i2_i3 HQs_i3_k3 -- contract in z direction. - // QHQ_k1_i2_k3 = HQs_i1_k1 HHQ_i1_i2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QHQ_k1_i2_k3 HQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(X, HQs, HH_Q); - MultAtB(HQs, H_HQ, Q_HQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < ndof1D; i2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += - Q_HQ(k1, i2 + k3*ndof1D) * HQs(i2, k2); - } - } - } - } - - // QQQ_k1_k2_k3 *= quad_data_k1_k2_k3 -- scaling with quadrature values. - double *d = quad_data->rho0DetJ0w.GetData() + z*nqp; - for (int q = 0; q < nqp; q++) { qqq[q] *= d[q]; } - - // QHQ_k1_i2_k3 = QQQ_k1_k2_k3 HQs_i2_k2 -- contract in y direction. - // The first step does some reordering (it's not product of matrices). - // HHQ_i1_i2_k3 = HQs_i1_k1 QHQ_k1_i2_k3 -- contract in x direction. - // Y_i1_i2_i3 = HHQ_i1_i2_k3 HQs_i3_k3 -- contract in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int i2 = 0; i2 < ndof1D; i2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_HQ(k1, i2 + ndof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_HQ(k1, i2 + ndof1D*k3) += - QQ_Q(k1 + nqp1D*k2, k3) * HQs(i2, k2); - } - } - } - } - mfem::Mult(HQs, Q_HQ, H_HQ); - MultABt(HH_Q, HQs, Y); - - for (int j = 0; j < yz.Size(); j++) - { - y[dofs[dof_map[j]]] += yz[j]; - } - } -} - -void LocalMassPAOperator::Mult(const Vector &x, Vector &y) const -{ - if (dim == 2) { MultQuad(x, y); } - else if (dim == 3) { MultHex(x, y); } - else { MFEM_ABORT("Unsupported dimension"); } -} - -// L2 mass matrix action on a single quadrilateral element in 2D. -void LocalMassPAOperator::MultQuad(const Vector &x, Vector &y) const -{ - const DenseMatrix &LQs = tensors1D->LQshape1D; - - y.SetSize(x.Size()); - y = 0.0; - - const int ndof1D = LQs.Height(), nqp1D = LQs.Width(); - DenseMatrix LQ(ndof1D, nqp1D), QQ(nqp1D, nqp1D); - DenseMatrix X(x.GetData(), ndof1D, ndof1D), Y(y.GetData(), ndof1D, ndof1D); - double *qq = QQ.GetData(); - const int nqp = nqp1D * nqp1D; - - // LQ_i1_k2 = X_i1_i2 LQs_i2_k2 -- contract in y direction. - // QQ_k1_k2 = LQs_i1_k1 LQ_i1_k2 -- contract in x direction. - mfem::Mult(X, LQs, LQ); - MultAtB(LQs, LQ, QQ); - - // QQ_k1_k2 *= quad_data_k1_k2 -- scaling with quadrature values. - const double *d = quad_data->rho0DetJ0w.GetData() + zone_id*nqp; - for (int q = 0; q < nqp; q++) { qq[q] *= d[q]; } - - // LQ_i1_k2 = LQs_i1_k1 QQ_k1_k2 -- contract in x direction. - // Y_i1_i2 = LQ_i1_k2 LQs_i2_k2 -- contract in y direction. - mfem::Mult(LQs, QQ, LQ); - MultABt(LQ, LQs, Y); -} - -// L2 mass matrix action on a single hexahedral element in 3D. -void LocalMassPAOperator::MultHex(const Vector &x, Vector &y) const -{ - const DenseMatrix &LQs = tensors1D->LQshape1D; - - y.SetSize(x.Size()); - y = 0.0; - - const int ndof1D = LQs.Height(), nqp1D = LQs.Width(); - DenseMatrix LL_Q(ndof1D * ndof1D, nqp1D); - DenseMatrix L_LQ(LL_Q.GetData(), ndof1D, ndof1D*nqp1D); - DenseMatrix Q_LQ(nqp1D, ndof1D*nqp1D); - DenseMatrix QQ_Q(nqp1D*nqp1D, nqp1D); - double *qqq = QQ_Q.GetData(); - DenseMatrix X(x.GetData(), ndof1D*ndof1D, ndof1D), - Y(y.GetData(), ndof1D*ndof1D, ndof1D); - const int nqp = nqp1D * nqp1D * nqp1D; - - // LLQ_i1_i2_k3 = X_i1_i2_i3 LQs_i3_k3 -- contract in z direction. - // QLQ_k1_i2_k3 = LQs_i1_k1 LLQ_i1_i2_k3 -- contract in x direction. - // QQQ_k1_k2_k3 = QLQ_k1_i2_k3 LQs_i2_k2 -- contract in y direction. - // The last step does some reordering (it's not product of matrices). - mfem::Mult(X, LQs, LL_Q); - MultAtB(LQs, L_LQ, Q_LQ); - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int k2 = 0; k2 < nqp1D; k2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - QQ_Q(k1 + nqp1D*k2, k3) = 0.0; - for (int i2 = 0; i2 < ndof1D; i2++) - { - QQ_Q(k1 + nqp1D*k2, k3) += - Q_LQ(k1, i2 + k3*ndof1D) * LQs(i2, k2); - } - } - } - } - - // QQQ_k1_k2_k3 *= quad_data_k1_k2_k3 -- scaling with quadrature values. - double *d = quad_data->rho0DetJ0w.GetData() + zone_id*nqp; - for (int q = 0; q < nqp; q++) { qqq[q] *= d[q]; } - - // QLQ_k1_i2_k3 = QQQ_k1_k2_k3 LQs_i2_k2 -- contract in y direction. - // The first step does some reordering (it's not product of matrices). - // LLQ_i1_i2_k3 = LQs_i1_k1 QLQ_k1_i2_k3 -- contract in x direction. - // Y_i1_i2_i3 = LLQ_i1_i2_k3 DQs_i3_k3 -- contract in z direction. - for (int k1 = 0; k1 < nqp1D; k1++) - { - for (int i2 = 0; i2 < ndof1D; i2++) - { - for (int k3 = 0; k3 < nqp1D; k3++) - { - Q_LQ(k1, i2 + ndof1D*k3) = 0.0; - for (int k2 = 0; k2 < nqp1D; k2++) - { - Q_LQ(k1, i2 + ndof1D*k3) += - QQ_Q(k1 + nqp1D*k2, k3) * LQs(i2, k2); - } - } - } - } - mfem::Mult(LQs, Q_LQ, L_LQ); - MultABt(LL_Q, LQs, Y); -} - -} // namespace hydrodynamics - -} // namespace mfem - -#endif // MFEM_USE_MPI diff --git a/amr/laghos_assembly.hpp b/amr/laghos_assembly.hpp deleted file mode 100644 index 012c277f..00000000 --- a/amr/laghos_assembly.hpp +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. - -#ifndef MFEM_LAGHOS_ASSEMBLY -#define MFEM_LAGHOS_ASSEMBLY - -#include "mfem.hpp" - - -#ifdef MFEM_USE_MPI - -#include -#include - -namespace mfem -{ - -namespace hydrodynamics -{ - -// Container for all data needed at quadrature points. -struct QuadratureData -{ - // TODO: use QuadratureFunctions? - - // Reference to physical Jacobian for the initial mesh. These are computed - // only at time zero and stored here. - DenseTensor Jac0inv; - - // Quadrature data used for full/partial assembly of the force operator. At - // each quadrature point, it combines the stress, inverse Jacobian, - // determinant of the Jacobian and the integration weight. It must be - // recomputed in every time step. - DenseTensor stressJinvT; - - // Quadrature data used for full/partial assembly of the mass matrices. At - // time zero, we compute and store (rho0 * det(J0) * qp_weight) at each - // quadrature point. Note the at any other time, we can compute - // rho = rho0 * det(J0) / det(J), representing the notion of pointwise mass - // conservation. - Vector rho0DetJ0w; - - // Initial length scale. This represents a notion of local mesh size. We - // assume that all initial zones have similar size. - double h0; - - // Estimate of the minimum time step over all quadrature points. This is - // recomputed at every time step to achieve adaptive time stepping. - double dt_est; - - QuadratureData(int dim, int nzones, int quads_per_zone) - : Jac0inv(dim, dim, nzones * quads_per_zone), - stressJinvT(nzones * quads_per_zone, dim, dim), - rho0DetJ0w(nzones * quads_per_zone) { } - - void Resize(int dim, int nzones, int quads_per_zone) - { - Jac0inv.SetSize(dim, dim, nzones * quads_per_zone); - stressJinvT.SetSize(nzones * quads_per_zone, dim, dim); - rho0DetJ0w.SetSize(nzones * quads_per_zone); - } -}; - -// Stores values of the one-dimensional shape functions and gradients at all 1D -// quadrature points. All sizes are (dofs1D_cnt x quads1D_cnt). -struct Tensors1D -{ - // H1 shape functions and gradients, L2 shape functions. - DenseMatrix HQshape1D, HQgrad1D, LQshape1D; - - Tensors1D(int H1order, int L2order, int nqp1D); -}; -extern const Tensors1D *tensors1D; - -class FastEvaluator -{ - const int dim; - ParFiniteElementSpace &H1FESpace; - -public: - FastEvaluator(ParFiniteElementSpace &h1fes) - : dim(h1fes.GetMesh()->Dimension()), H1FESpace(h1fes) { } - - void GetL2Values(const Vector &vecL2, Vector &vecQP) const; - // The input vec is an H1 function with dim components, over a zone. - // The output is J_ij = d(vec_i) / d(x_j) with ij = 1 .. dim. - void GetVectorGrad(const DenseMatrix &vec, DenseTensor &J) const; -}; -extern const FastEvaluator *evaluator; - -// This class is used only for visualization. It assembles (rho, phi) in each -// zone, which is used by LagrangianHydroOperator::ComputeDensity to do an L2 -// projection of the density. -class DensityIntegrator : public LinearFormIntegrator -{ -private: - const QuadratureData &quad_data; - -public: - DensityIntegrator(QuadratureData &quad_data_) : quad_data(quad_data_) { } - - virtual void AssembleRHSElementVect(const FiniteElement &fe, - ElementTransformation &Tr, - Vector &elvect); -}; - -// Assembles element contributions to the global force matrix. This class is -// used for the full assembly case; it's not used with partial assembly. -class ForceIntegrator : public BilinearFormIntegrator -{ -private: - const QuadratureData &quad_data; - -public: - ForceIntegrator(QuadratureData &quad_data_) : quad_data(quad_data_) { } - - virtual void AssembleElementMatrix2(const FiniteElement &trial_fe, - const FiniteElement &test_fe, - ElementTransformation &Trans, - DenseMatrix &elmat); -}; - -// Performs partial assembly, which corresponds to (and replaces) the use of the -// LagrangianHydroOperator::Force global matrix. -class ForcePAOperator : public Operator -{ -private: - const int dim, nzones; - - QuadratureData *quad_data; - ParFiniteElementSpace &H1FESpace, &L2FESpace; - - // Force matrix action on quadrilateral elements in 2D. - void MultQuad(const Vector &vecL2, Vector &vecH1) const; - // Force matrix action on hexahedral elements in 3D. - void MultHex(const Vector &vecL2, Vector &vecH1) const; - - // Transpose force matrix action on quadrilateral elements in 2D. - void MultTransposeQuad(const Vector &vecH1, Vector &vecL2) const; - // Transpose force matrix action on hexahedral elements in 3D. - void MultTransposeHex(const Vector &vecH1, Vector &vecL2) const; - -public: - ForcePAOperator(QuadratureData *quad_data_, - ParFiniteElementSpace &h1fes, ParFiniteElementSpace &l2fes) - : dim(h1fes.GetMesh()->Dimension()), nzones(h1fes.GetMesh()->GetNE()), - quad_data(quad_data_), H1FESpace(h1fes), L2FESpace(l2fes) { } - - virtual void Mult(const Vector &vecL2, Vector &vecH1) const; - virtual void MultTranspose(const Vector &vecH1, Vector &vecL2) const; - - ~ForcePAOperator() { } -}; - -// Performs partial assembly for the velocity mass matrix. -class MassPAOperator : public Operator -{ -private: - const int dim, nzones; - - QuadratureData *quad_data; - ParFiniteElementSpace &FESpace; - - // Mass matrix action on quadrilateral elements in 2D. - void MultQuad(const Vector &x, Vector &y) const; - // Mass matrix action on hexahedral elements in 3D. - void MultHex(const Vector &x, Vector &y) const; - -public: - MassPAOperator(QuadratureData *quad_data_, ParFiniteElementSpace &fes) - : Operator(fes.GetVSize()), - dim(fes.GetMesh()->Dimension()), nzones(fes.GetMesh()->GetNE()), - quad_data(quad_data_), FESpace(fes) - { } - - // Mass matrix action. - virtual void Mult(const Vector &x, Vector &y) const; - - virtual const Operator *GetProlongation() const - { return FESpace.GetProlongationMatrix(); } - virtual const Operator *GetRestriction() const - { return FESpace.GetRestrictionMatrix(); } -}; - -// Performs partial assembly for the energy mass matrix on a single zone. -// Used to perform local CG solves, thus avoiding unnecessary communication. -class LocalMassPAOperator : public Operator -{ -private: - const int dim; - int zone_id; - - QuadratureData *quad_data; - - // Mass matrix action on a quadrilateral element in 2D. - void MultQuad(const Vector &x, Vector &y) const; - // Mass matrix action on a hexahedral element in 3D. - void MultHex(const Vector &x, Vector &y) const; - -public: - LocalMassPAOperator(QuadratureData *quad_data_, ParFiniteElementSpace &fes) - : Operator(fes.GetFE(0)->GetDof()), - dim(fes.GetMesh()->Dimension()), zone_id(0), - quad_data(quad_data_) - { } - void SetZoneId(int zid) { zone_id = zid; } - - virtual void Mult(const Vector &x, Vector &y) const; -}; - -} // namespace hydrodynamics - -} // namespace mfem - -#endif // MFEM_USE_MPI - -#endif // MFEM_LAGHOS_ASSEMBLY diff --git a/amr/laghos_solver.cpp b/amr/laghos_solver.cpp deleted file mode 100644 index b50dd0e7..00000000 --- a/amr/laghos_solver.cpp +++ /dev/null @@ -1,737 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. - -#include "laghos_solver.hpp" - -#ifdef MFEM_USE_MPI - -using namespace std; - -namespace mfem -{ - -namespace hydrodynamics -{ - -void VisualizeField(socketstream &sock, const char *vishost, int visport, - ParGridFunction &gf, const char *title, - int x, int y, int w, int h, bool vec) -{ - ParMesh &pmesh = *gf.ParFESpace()->GetParMesh(); - MPI_Comm comm = pmesh.GetComm(); - - int num_procs, myid; - MPI_Comm_size(comm, &num_procs); - MPI_Comm_rank(comm, &myid); - - bool newly_opened = false; - int connection_failed; - - do - { - if (myid == 0) - { - if (!sock.is_open() || !sock) - { - sock.open(vishost, visport); - newly_opened = true; - } - sock << "solution\n"; - } - - sock.precision(8); - pmesh.PrintAsOne(sock); - gf.SaveAsOne(sock); - - if (myid == 0 && newly_opened) - { - const char* keys = (gf.FESpace()->GetMesh()->Dimension() == 2) - ? "mAcRjlPPPPPPPP" : "maaAcl"; - - sock << "window_title '" << title << "'\n" - << "window_geometry " - << x << " " << y << " " << w << " " << h << "\n" - << "keys " << keys; - if ( vec ) { sock << "vvv"; } - sock << endl; - } - - if (myid == 0) - { - connection_failed = !sock && !newly_opened; - } - MPI_Bcast(&connection_failed, 1, MPI_INT, 0, comm); - } - while (connection_failed); -} - -void VisualizeElementValues(socketstream &sock, const char *vishost, - int visport, - ParMesh* pmesh, Vector &values, const char *title, - int x, int y, int w, int h) -{ - L2_FECollection l2_fec(0, pmesh->Dimension()); - ParFiniteElementSpace l2_fes(pmesh, &l2_fec); - ParGridFunction l2_gf(&l2_fes); - l2_gf = values; - VisualizeField(sock, vishost, visport, l2_gf, title, x, y, w, h); -} - - -LagrangianHydroOperator::LagrangianHydroOperator(int size, - ParFiniteElementSpace &h1_fes, - ParFiniteElementSpace &l2_fes, - Array &essential_tdofs, - ParGridFunction &rho0, - int source_type_, double cfl_, - Coefficient *material_, - bool visc, bool pa, - double cgt, int cgiter) - : TimeDependentOperator(size), - H1FESpace(h1_fes), L2FESpace(l2_fes), - ess_tdofs(essential_tdofs), - nzones(h1_fes.GetMesh()->GetNE()), - dim(h1_fes.GetMesh()->Dimension()), - l2dofs_cnt(l2_fes.GetFE(0)->GetDof()), - h1dofs_cnt(h1_fes.GetFE(0)->GetDof()), - source_type(source_type_), cfl(cfl_), - use_viscosity(visc), p_assembly(pa), cg_rel_tol(cgt), cg_max_iter(cgiter), - material_pcf(material_), - rho0(rho0), - rho0_coeff(&rho0), - x0_gf(&h1_fes), - Mv(&h1_fes), Me_inv(l2dofs_cnt, l2dofs_cnt, nzones), - integ_rule(IntRules.Get(h1_fes.GetMesh()->GetElementBaseGeometry(0), - 3*h1_fes.GetOrder(0) + l2_fes.GetOrder(0) - 1)), - quad_data(dim, nzones, integ_rule.GetNPoints()), - quad_data_is_current(false), - Force(&l2_fes, &h1_fes), ForcePA(&quad_data, h1_fes, l2_fes), - VMassPA(&quad_data, H1FESpace), locEMassPA(&quad_data, l2_fes), - locCG(), timer() -{ - // Standard local assembly and inversion for energy mass matrices. - DenseMatrix Me(l2dofs_cnt); - DenseMatrixInverse inv(&Me); - MassIntegrator mi(rho0_coeff, &integ_rule); - for (int i = 0; i < nzones; i++) - { - mi.AssembleElementMatrix(*l2_fes.GetFE(i), - *l2_fes.GetElementTransformation(i), Me); - inv.Factor(); - inv.GetInverseMatrix(Me_inv(i)); - } - - // Standard assembly for the velocity mass matrix. - VectorMassIntegrator *vmi = new VectorMassIntegrator(rho0_coeff, &integ_rule); - Mv.AddDomainIntegrator(vmi); - Mv.Assemble(); - - // Values of rho0DetJ0 and Jac0inv at all quadrature points. - const int nqp = integ_rule.GetNPoints(); - Vector rho_vals(nqp); - for (int i = 0; i < nzones; i++) - { - rho0.GetValues(i, integ_rule, rho_vals); - ElementTransformation *T = h1_fes.GetElementTransformation(i); - for (int q = 0; q < nqp; q++) - { - const IntegrationPoint &ip = integ_rule.IntPoint(q); - T->SetIntPoint(&ip); - - DenseMatrixInverse Jinv(T->Jacobian()); - Jinv.GetInverseMatrix(quad_data.Jac0inv(i*nqp + q)); - - const double rho0DetJ0 = T->Weight() * rho_vals(q); - quad_data.rho0DetJ0w(i*nqp + q) = rho0DetJ0 * - integ_rule.IntPoint(q).weight; - } - } - - // Save initial (undeformed) mesh configuration for use in AMRUpdate later. - x0_gf = *(h1_fes.GetMesh()->GetNodes()); - - // Initial local mesh size (assumes all mesh elements are of the same type). - double loc_area = 0.0, glob_area; - int loc_z_cnt = nzones, glob_z_cnt; - ParMesh *pm = H1FESpace.GetParMesh(); - for (int i = 0; i < nzones; i++) { loc_area += pm->GetElementVolume(i); } - MPI_Allreduce(&loc_area, &glob_area, 1, MPI_DOUBLE, MPI_SUM, pm->GetComm()); - MPI_Allreduce(&loc_z_cnt, &glob_z_cnt, 1, MPI_INT, MPI_SUM, pm->GetComm()); - switch (pm->GetElementBaseGeometry(0)) - { - case Geometry::SEGMENT: - quad_data.h0 = glob_area / glob_z_cnt; break; - case Geometry::SQUARE: - quad_data.h0 = sqrt(glob_area / glob_z_cnt); break; - case Geometry::TRIANGLE: - quad_data.h0 = sqrt(2.0 * glob_area / glob_z_cnt); break; - case Geometry::CUBE: - quad_data.h0 = pow(glob_area / glob_z_cnt, 1.0/3.0); break; - case Geometry::TETRAHEDRON: - quad_data.h0 = pow(6.0 * glob_area / glob_z_cnt, 1.0/3.0); break; - default: MFEM_ABORT("Unknown zone type!"); - } - quad_data.h0 /= (double) H1FESpace.GetOrder(0); - - ForceIntegrator *fi = new ForceIntegrator(quad_data); - fi->SetIntRule(&integ_rule); - Force.AddDomainIntegrator(fi); - // Make a dummy assembly to figure out the sparsity. - Force.Assemble(0); - Force.Finalize(0); - - if (p_assembly) - { - tensors1D = new Tensors1D(H1FESpace.GetFE(0)->GetOrder(), - L2FESpace.GetFE(0)->GetOrder(), - int(floor(0.7 + pow(nqp, 1.0 / dim)))); - evaluator = new FastEvaluator(H1FESpace); - } - - locCG.SetOperator(locEMassPA); - locCG.iterative_mode = false; - locCG.SetRelTol(1e-8); - locCG.SetAbsTol(1e-8 * numeric_limits::epsilon()); - locCG.SetMaxIter(200); - locCG.SetPrintLevel(0); -} - -void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const -{ - dS_dt = 0.0; - - // Make sure that the mesh positions correspond to the ones in S. This is - // needed only because some mfem time integrators don't update the solution - // vector at every intermediate stage (hence they don't change the mesh). - Vector* sptr = (Vector*) &S; - ParGridFunction x; - x.MakeRef(&H1FESpace, *sptr, 0); - H1FESpace.GetParMesh()->NewNodes(x, false); - - UpdateQuadratureData(S); - - // The monolithic BlockVector stores the unknown fields as follows: - // - Position - // - Velocity - // - Specific Internal Energy - - const int VsizeL2 = L2FESpace.GetVSize(); - const int VsizeH1 = H1FESpace.GetVSize(); - - ParGridFunction v, e; - v.MakeRef(&H1FESpace, *sptr, VsizeH1); - e.MakeRef(&L2FESpace, *sptr, VsizeH1*2); - - ParGridFunction dx, dv, de; - dx.MakeRef(&H1FESpace, dS_dt, 0); - dv.MakeRef(&H1FESpace, dS_dt, VsizeH1); - de.MakeRef(&L2FESpace, dS_dt, VsizeH1*2); - - // Set dx_dt = v (explicit). - dx = v; - - if (!p_assembly) - { - //Force = 0.0; - Force.Update(); - timer.sw_force.Start(); - Force.Assemble(); - timer.sw_force.Stop(); - } - - // Solve for velocity. - Vector one(VsizeL2), rhs(VsizeH1), B, X; one = 1.0; - if (p_assembly) - { - timer.sw_force.Start(); - ForcePA.Mult(one, rhs); - timer.sw_force.Stop(); - rhs.Neg(); - - Operator *cVMassPA; - VMassPA.FormLinearSystem(ess_tdofs, dv, rhs, cVMassPA, X, B); - CGSolver cg(H1FESpace.GetParMesh()->GetComm()); - cg.SetOperator(*cVMassPA); - cg.SetRelTol(cg_rel_tol); cg.SetAbsTol(0.0); - cg.SetMaxIter(cg_max_iter); - cg.SetPrintLevel(0); - timer.sw_cgH1.Start(); - cg.Mult(B, X); - timer.sw_cgH1.Stop(); - timer.H1cg_iter += cg.GetNumIterations(); - VMassPA.RecoverFEMSolution(X, rhs, dv); - delete cVMassPA; - } - else - { - timer.sw_force.Start(); - Force.Mult(one, rhs); - timer.sw_force.Stop(); - rhs.Neg(); - - HypreParMatrix A; - Mv.FormLinearSystem(ess_tdofs, dv, rhs, A, X, B); - CGSolver cg(H1FESpace.GetParMesh()->GetComm()); - cg.SetOperator(A); - cg.SetRelTol(cg_rel_tol); cg.SetAbsTol(0.0); - cg.SetMaxIter(cg_max_iter); - cg.SetPrintLevel(-1); - timer.sw_cgH1.Start(); - cg.Mult(B, X); - timer.sw_cgH1.Stop(); - timer.H1cg_iter += cg.GetNumIterations(); - Mv.RecoverFEMSolution(X, rhs, dv); - } - - // Solve for energy, assemble the energy source if such exists. - LinearForm *e_source = NULL; - if (source_type == 1) // 2D Taylor-Green. - { - e_source = new LinearForm(&L2FESpace); - TaylorCoefficient coeff; - DomainLFIntegrator *d = new DomainLFIntegrator(coeff, &integ_rule); - e_source->AddDomainIntegrator(d); - e_source->Assemble(); - } - Array l2dofs; - Vector e_rhs(VsizeL2), loc_rhs(l2dofs_cnt), loc_de(l2dofs_cnt); - if (p_assembly) - { - timer.sw_force.Start(); - ForcePA.MultTranspose(v, e_rhs); - timer.sw_force.Stop(); - - if (e_source) { e_rhs += *e_source; } - for (int z = 0; z < nzones; z++) - { - L2FESpace.GetElementDofs(z, l2dofs); - e_rhs.GetSubVector(l2dofs, loc_rhs); - locEMassPA.SetZoneId(z); - timer.sw_cgL2.Start(); - locCG.Mult(loc_rhs, loc_de); - timer.sw_cgL2.Stop(); - timer.L2dof_iter += locCG.GetNumIterations() * l2dofs_cnt; - de.SetSubVector(l2dofs, loc_de); - } - } - else - { - timer.sw_force.Start(); - Force.MultTranspose(v, e_rhs); - timer.sw_force.Stop(); - if (e_source) { e_rhs += *e_source; } - for (int z = 0; z < nzones; z++) - { - L2FESpace.GetElementDofs(z, l2dofs); - e_rhs.GetSubVector(l2dofs, loc_rhs); - timer.sw_cgL2.Start(); - Me_inv(z).Mult(loc_rhs, loc_de); - timer.sw_cgL2.Stop(); - timer.L2dof_iter += l2dofs_cnt; - de.SetSubVector(l2dofs, loc_de); - } - } - delete e_source; - - quad_data_is_current = false; -} - -double LagrangianHydroOperator::GetTimeStepEstimate(const Vector &S) const -{ - Vector* sptr = (Vector*) &S; - ParGridFunction x; - x.MakeRef(&H1FESpace, *sptr, 0); - H1FESpace.GetParMesh()->NewNodes(x, false); - UpdateQuadratureData(S); - - double glob_dt_est; - MPI_Allreduce(&quad_data.dt_est, &glob_dt_est, 1, MPI_DOUBLE, MPI_MIN, - H1FESpace.GetParMesh()->GetComm()); - return glob_dt_est; -} - -void LagrangianHydroOperator::ResetTimeStepEstimate() const -{ - quad_data.dt_est = numeric_limits::infinity(); -} - -void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) -{ - rho.SetSpace(&L2FESpace); - - DenseMatrix Mrho(l2dofs_cnt); - Vector rhs(l2dofs_cnt), rho_z(l2dofs_cnt); - Array dofs(l2dofs_cnt); - DenseMatrixInverse inv(&Mrho); - - MassIntegrator mi(&integ_rule); - DensityIntegrator di(quad_data); - di.SetIntRule(&integ_rule); - - for (int i = 0; i < nzones; i++) - { - di.AssembleRHSElementVect(*L2FESpace.GetFE(i), - *L2FESpace.GetElementTransformation(i), rhs); - mi.AssembleElementMatrix(*L2FESpace.GetFE(i), - *L2FESpace.GetElementTransformation(i), Mrho); - inv.Factor(); - inv.Mult(rhs, rho_z); - - L2FESpace.GetElementDofs(i, dofs); - rho.SetSubVector(dofs, rho_z); - } -} - -void LagrangianHydroOperator::PrintTimingData(bool IamRoot, int steps) -{ - double my_rt[5], rt_max[5]; - my_rt[0] = timer.sw_cgH1.RealTime(); - my_rt[1] = timer.sw_cgL2.RealTime(); - my_rt[2] = timer.sw_force.RealTime(); - my_rt[3] = timer.sw_qdata.RealTime(); - my_rt[4] = my_rt[0] + my_rt[2] + my_rt[3]; - MPI_Reduce(my_rt, rt_max, 5, MPI_DOUBLE, MPI_MAX, 0, H1FESpace.GetComm()); - - HYPRE_Int mydata[2], alldata[2]; - mydata[0] = timer.L2dof_iter; - mydata[1] = timer.quad_tstep; - MPI_Reduce(mydata, alldata, 2, HYPRE_MPI_INT, MPI_SUM, 0, - H1FESpace.GetComm()); - - if (IamRoot) - { - const HYPRE_Int H1gsize = H1FESpace.GlobalTrueVSize(), - L2gsize = L2FESpace.GlobalTrueVSize(); - using namespace std; - cout << endl; - cout << "CG (H1) total time: " << rt_max[0] << endl; - cout << "CG (H1) rate (megadofs x cg_iterations / second): " - << 1e-6 * H1gsize * timer.H1cg_iter / rt_max[0] << endl; - cout << endl; - cout << "CG (L2) total time: " << rt_max[1] << endl; - cout << "CG (L2) rate (megadofs x cg_iterations / second): " - << 1e-6 * alldata[0] / rt_max[1] << endl; - cout << endl; - // The Force operator is applied twice per time step, on the H1 and the L2 - // vectors, respectively. - cout << "Forces total time: " << rt_max[2] << endl; - cout << "Forces rate (megadofs x timesteps / second): " - << 1e-6 * steps * (H1gsize + L2gsize) / rt_max[2] << endl; - cout << endl; - cout << "UpdateQuadData total time: " << rt_max[3] << endl; - cout << "UpdateQuadData rate (megaquads x timesteps / second): " - << 1e-6 * alldata[1] * integ_rule.GetNPoints() / rt_max[3] << endl; - cout << endl; - cout << "Major kernels total time (seconds): " << rt_max[4] << endl; - cout << "Major kernels total rate (megadofs x time steps / second): " - << 1e-6 * H1gsize * steps / rt_max[4] << endl; - } -} - -LagrangianHydroOperator::~LagrangianHydroOperator() -{ - delete tensors1D; -} - -void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const -{ - if (quad_data_is_current) { return; } - timer.sw_qdata.Start(); - - const int nqp = integ_rule.GetNPoints(); - - ParGridFunction x, v, e; - Vector* sptr = (Vector*) &S; - x.MakeRef(&H1FESpace, *sptr, 0); - v.MakeRef(&H1FESpace, *sptr, H1FESpace.GetVSize()); - e.MakeRef(&L2FESpace, *sptr, 2*H1FESpace.GetVSize()); - Vector e_vals, e_loc(l2dofs_cnt), vector_vals(h1dofs_cnt * dim); - DenseMatrix Jpi(dim), sgrad_v(dim), Jinv(dim), stress(dim), stressJiT(dim), - vecvalMat(vector_vals.GetData(), h1dofs_cnt, dim); - DenseTensor grad_v_ref(dim, dim, nqp); - Array L2dofs, H1dofs; - - zone_max_visc.SetSize(nzones); - zone_max_visc = 0.0; - - zone_vgrad.SetSize(nzones); - zone_vgrad = 0.0; - - // Batched computations are needed, because hydrodynamic codes usually - // involve expensive computations of material properties. Although this - // miniapp uses simple EOS equations, we still want to represent the batched - // cycle structure. - int nzones_batch = 3; - const int nbatches = nzones / nzones_batch + 1; // +1 for the remainder. - int nqp_batch = nqp * nzones_batch; - double *gamma_b = new double[nqp_batch], - *rho_b = new double[nqp_batch], - *e_b = new double[nqp_batch], - *p_b = new double[nqp_batch], - *cs_b = new double[nqp_batch]; - // Jacobians of reference->physical transformations for all quadrature points - // in the batch. - DenseTensor *Jpr_b = new DenseTensor[nqp_batch]; - for (int b = 0; b < nbatches; b++) - { - int z_id = b * nzones_batch; // Global index over zones. - // The last batch might not be full. - if (z_id == nzones) { break; } - else if (z_id + nzones_batch > nzones) - { - nzones_batch = nzones - z_id; - nqp_batch = nqp * nzones_batch; - } - - double min_detJ = numeric_limits::infinity(); - for (int z = 0; z < nzones_batch; z++) - { - ElementTransformation *T = H1FESpace.GetElementTransformation(z_id); - Jpr_b[z].SetSize(dim, dim, nqp); - - if (p_assembly) - { - // Energy values at quadrature point. - L2FESpace.GetElementDofs(z_id, L2dofs); - e.GetSubVector(L2dofs, e_loc); - evaluator->GetL2Values(e_loc, e_vals); - - // All reference->physical Jacobians at the quadrature points. - H1FESpace.GetElementVDofs(z_id, H1dofs); - x.GetSubVector(H1dofs, vector_vals); - evaluator->GetVectorGrad(vecvalMat, Jpr_b[z]); - } - else - { - e.GetValues(z_id, integ_rule, e_vals); - } - - for (int q = 0; q < nqp; q++) - { - const IntegrationPoint &ip = integ_rule.IntPoint(q); - T->SetIntPoint(&ip); - if (!p_assembly) { Jpr_b[z](q) = T->Jacobian(); } - const double detJ = Jpr_b[z](q).Det(); - min_detJ = min(min_detJ, detJ); - - const int idx = z * nqp + q; - if (material_pcf == NULL) { gamma_b[idx] = 5./3.; } // Ideal gas. - else { gamma_b[idx] = material_pcf->Eval(*T, ip); } - rho_b[idx] = quad_data.rho0DetJ0w(z_id*nqp + q) / detJ / ip.weight; - e_b[idx] = max(0.0, e_vals(q)); - } - ++z_id; - } - - // Batched computation of material properties. - ComputeMaterialProperties(nqp_batch, gamma_b, rho_b, e_b, p_b, cs_b); - - z_id -= nzones_batch; - for (int z = 0; z < nzones_batch; z++) - { - ElementTransformation *T = H1FESpace.GetElementTransformation(z_id); - - if (p_assembly) - { - // All reference->physical Jacobians at the quadrature points. - H1FESpace.GetElementVDofs(z_id, H1dofs); - v.GetSubVector(H1dofs, vector_vals); - evaluator->GetVectorGrad(vecvalMat, grad_v_ref); - } - for (int q = 0; q < nqp; q++) - { - const IntegrationPoint &ip = integ_rule.IntPoint(q); - T->SetIntPoint(&ip); - // Note that the Jacobian was already computed above. We've chosen - // not to store the Jacobians for all batched quadrature points. - const DenseMatrix &Jpr = Jpr_b[z](q); - CalcInverse(Jpr, Jinv); - const double detJ = Jpr.Det(), rho = rho_b[z*nqp + q], - p = p_b[z*nqp + q], sound_speed = cs_b[z*nqp + q]; - - stress = 0.0; - for (int d = 0; d < dim; d++) { stress(d, d) = -p; } - - double visc_coeff = 0.0, det_v_grad = 0.0; - if (use_viscosity) - { - // Compression-based length scale at the point. The first - // eigenvector of the symmetric velocity gradient gives the - // direction of maximal compression. This is used to define the - // relative change of the initial length scale. - if (p_assembly) - { - mfem::Mult(grad_v_ref(q), Jinv, sgrad_v); - } - else - { - v.GetVectorGradient(*T, sgrad_v); - } - sgrad_v.Symmetrize(); - det_v_grad = sgrad_v.Det(); - - double eig_val_data[3], eig_vec_data[9]; - if (dim==1) - { - eig_val_data[0] = sgrad_v(0, 0); - eig_vec_data[0] = 1.; - } - else { sgrad_v.CalcEigenvalues(eig_val_data, eig_vec_data); } - Vector compr_dir(eig_vec_data, dim); - // Computes the initial->physical transformation Jacobian. - mfem::Mult(Jpr, quad_data.Jac0inv(z_id*nqp + q), Jpi); - Vector ph_dir(dim); Jpi.Mult(compr_dir, ph_dir); - // Change of the initial mesh size in the compression direction. - double h0 = quad_data.h0; - ParNCMesh* pncmesh = H1FESpace.GetParMesh()->pncmesh; - if (pncmesh) - { - // TODO: h0 should be a smooth function - h0 /= (1 << pncmesh->GetElementDepth(z_id)); - } - const double h = h0 * ph_dir.Norml2() / compr_dir.Norml2(); - - // Measure of maximal compression. - const double mu = eig_val_data[0]; - visc_coeff = 2.0 * rho * h * h * fabs(mu); - if (mu < 0.0) { visc_coeff += 0.5 * rho * h * sound_speed; } - stress.Add(visc_coeff, sgrad_v); - } - - // Time step estimate at the point. Here the more relevant length - // scale is related to the actual mesh deformation; we use the min - // singular value of the ref->physical Jacobian. In addition, the - // time step estimate should be aware of the presence of shocks. - const double h_min = - Jpr.CalcSingularvalue(dim-1) / (double) H1FESpace.GetOrder(0); - const double inv_dt = sound_speed / h_min + - 2.5 * visc_coeff / rho / h_min / h_min; - if (min_detJ < 0.0) - { - // This will force repetition of the step with smaller dt. - quad_data.dt_est = 0.0; - } - else - { - quad_data.dt_est = min(quad_data.dt_est, cfl * (1.0 / inv_dt) ); - } - - // Quadrature data for partial assembly of the force operator. - MultABt(stress, Jinv, stressJiT); - stressJiT *= integ_rule.IntPoint(q).weight * detJ; - for (int vd = 0 ; vd < dim; vd++) - { - for (int gd = 0; gd < dim; gd++) - { - quad_data.stressJinvT(vd)(z_id*nqp + q, gd) = - stressJiT(vd, gd); - } - } - - // Track maximum artificial viscosity per zone - zone_max_visc(z_id) = std::max(visc_coeff, zone_max_visc(z_id)); - zone_vgrad(z_id) = std::max(std::abs(det_v_grad), zone_vgrad(z_id)); - //zone_vgrad(z_id) += std::abs(det_v_grad); - } - ++z_id; - } - } - delete [] gamma_b; - delete [] rho_b; - delete [] e_b; - delete [] p_b; - delete [] cs_b; - delete [] Jpr_b; - quad_data_is_current = true; - - timer.sw_qdata.Stop(); - timer.quad_tstep += nzones; -} - -void LagrangianHydroOperator::AMRUpdate(const Vector &S, bool quick) -{ - ParMesh *pmesh = H1FESpace.GetParMesh(); - - width = height = S.Size(); - nzones = pmesh->GetNE(); - - x0_gf.Update(); - rho0.Update(); - - if (quick) { return; } - - // go back to initial mesh configuration temporarily - int own_nodes = 0; - GridFunction *x_gf = &x0_gf; - pmesh->SwapNodes(x_gf, own_nodes); - - // update mass matrix - // TODO: don't reassemble everything! - Mv.Update(); - Mv.Assemble(); - - // update Me_inv - // TODO: do this better too - { - Me_inv.SetSize(l2dofs_cnt, l2dofs_cnt, nzones); - - DenseMatrix Me(l2dofs_cnt); - DenseMatrixInverse inv(&Me); - MassIntegrator mi(rho0_coeff, &integ_rule); - for (int i = 0; i < nzones; i++) - { - mi.AssembleElementMatrix(*L2FESpace.GetFE(i), - *L2FESpace.GetElementTransformation(i), Me); - inv.Factor(); - inv.GetInverseMatrix(Me_inv(i)); - } - } - - // resize quadrature data and make sure 'stressJinvT' will be recomputed - quad_data.Resize(dim, nzones, integ_rule.GetNPoints()); - quad_data_is_current = false; - - // update 'rho0DetJ0' and 'Jac0inv' at all quadrature points - // TODO: remove code duplication - const int nqp = integ_rule.GetNPoints(); - Vector rho_vals(nqp); - for (int i = 0; i < nzones; i++) - { - rho0.GetValues(i, integ_rule, rho_vals); - ElementTransformation *T = H1FESpace.GetElementTransformation(i); - for (int q = 0; q < nqp; q++) - { - const IntegrationPoint &ip = integ_rule.IntPoint(q); - T->SetIntPoint(&ip); - - DenseMatrixInverse Jinv(T->Jacobian()); - Jinv.GetInverseMatrix(quad_data.Jac0inv(i*nqp + q)); - - const double rho0DetJ0 = T->Weight() * rho_vals(q); - quad_data.rho0DetJ0w(i*nqp + q) = rho0DetJ0 * - integ_rule.IntPoint(q).weight; - } - } - - // swap back to deformed mesh configuration - pmesh->SwapNodes(x_gf, own_nodes); -} - -} // namespace hydrodynamics - -} // namespace mfem - -#endif // MFEM_USE_MPI diff --git a/amr/laghos_solver.hpp b/amr/laghos_solver.hpp deleted file mode 100644 index 747547c5..00000000 --- a/amr/laghos_solver.hpp +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -// reserved. See files LICENSE and NOTICE for details. -// -// This file is part of CEED, a collection of benchmarks, miniapps, software -// libraries and APIs for efficient high-order finite element and spectral -// element discretizations for exascale applications. For more information and -// source code availability see http://github.com/ceed. -// -// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -// a collaborative effort of two U.S. Department of Energy organizations (Office -// of Science and the National Nuclear Security Administration) responsible for -// the planning and preparation of a capable exascale ecosystem, including -// software, applications, hardware, advanced system engineering and early -// testbed platforms, in support of the nation's exascale computing imperative. - -#ifndef MFEM_LAGHOS_SOLVER -#define MFEM_LAGHOS_SOLVER - -#include "mfem.hpp" -#include "laghos_assembly.hpp" - -#ifdef MFEM_USE_MPI - -#include -#include -#include - -namespace mfem -{ - -namespace hydrodynamics -{ - -/// Visualize the given parallel grid function, using a GLVis server on the -/// specified host and port. Set the visualization window title, and optionally, -/// its geometry. -void VisualizeField(socketstream &sock, const char *vishost, int visport, - ParGridFunction &gf, const char *title, - int x = 0, int y = 0, int w = 400, int h = 400, - bool vec = false); - -/// Visualize per element scalar values. -void VisualizeElementValues(socketstream &sock, const char *vishost, - int visport, - ParMesh* pmesh, Vector &values, const char *title, - int x, int y, int w, int h); - -// These are defined in laghos.cpp -double rho0(const Vector &); -void v0(const Vector &, Vector &); -double e0(const Vector &); -double gamma(const Vector &); - -struct TimingData -{ - // Total times for all major computations: - // CG solves (H1 and L2) / force RHS assemblies / quadrature computations. - StopWatch sw_cgH1, sw_cgL2, sw_force, sw_qdata; - - // These accumulate the total processed dofs or quad points: - // #(CG iterations) for the H1 CG solve. - // #dofs * #(CG iterations) for the L2 CG solve. - // #quads * #(RK sub steps) for the quadrature data computations. - int H1cg_iter, L2dof_iter, quad_tstep; - - TimingData() - : H1cg_iter(0), L2dof_iter(0), quad_tstep(0) { } -}; - -// Given a solutions state (x, v, e), this class performs all necessary -// computations to evaluate the new slopes (dx_dt, dv_dt, de_dt). -class LagrangianHydroOperator : public TimeDependentOperator -{ -protected: - ParFiniteElementSpace &H1FESpace; - ParFiniteElementSpace &L2FESpace; - - Array &ess_tdofs; - - int nzones; - const int dim, l2dofs_cnt, h1dofs_cnt, source_type; - const double cfl; - const bool use_viscosity, p_assembly; - const double cg_rel_tol; - const int cg_max_iter; - Coefficient *material_pcf; - - ParGridFunction &rho0; - GridFunctionCoefficient rho0_coeff; // TODO: remove when Mv update improved - ParGridFunction x0_gf; // copy of initial mesh position - - // Velocity mass matrix and local inverses of the energy mass matrices. These - // are constant in time, due to the pointwise mass conservation property. - mutable ParBilinearForm Mv; - DenseTensor Me_inv; - - // Integration rule for all assemblies. - const IntegrationRule &integ_rule; - - // Data associated with each quadrature point in the mesh. These values are - // recomputed at each time step. - mutable QuadratureData quad_data; - mutable bool quad_data_is_current; - - // Force matrix that combines the kinematic and thermodynamic spaces. It is - // assembled in each time step and then it is used to compute the final - // right-hand sides for momentum and specific internal energy. - mutable MixedBilinearForm Force; - - // Same as above, but done through partial assembly. - ForcePAOperator ForcePA; - - // Mass matrices done through partial assembly: - // velocity (coupled H1 assembly) and energy (local L2 assemblies). - mutable MassPAOperator VMassPA; - mutable LocalMassPAOperator locEMassPA; - - // Linear solver for energy. - CGSolver locCG; - - mutable Vector zone_max_visc, zone_vgrad; - - mutable TimingData timer; - - void ComputeMaterialProperties(int nvalues, const double gamma[], - const double rho[], const double e[], - double p[], double cs[]) const - { - for (int v = 0; v < nvalues; v++) - { - p[v] = (gamma[v] - 1.0) * rho[v] * e[v]; - cs[v] = sqrt(gamma[v] * (gamma[v]-1.0) * e[v]); - } - } - - void UpdateQuadratureData(const Vector &S) const; - -public: - LagrangianHydroOperator(int size, ParFiniteElementSpace &h1_fes, - ParFiniteElementSpace &l2_fes, - Array &essential_tdofs, ParGridFunction &rho0, - int source_type_, double cfl_, - Coefficient *material_, bool visc, bool pa, - double cgt, int cgiter); - - // Solve for dx_dt, dv_dt and de_dt. - virtual void Mult(const Vector &S, Vector &dS_dt) const; - - // Calls UpdateQuadratureData to compute the new quad_data.dt_estimate. - double GetTimeStepEstimate(const Vector &S) const; - void ResetTimeStepEstimate() const; - void ResetQuadratureData() const { quad_data_is_current = false; } - - // The density values, which are stored only at some quadrature points, are - // projected as a ParGridFunction. - void ComputeDensity(ParGridFunction &rho); - - // Update all internal data on mesh change. - void AMRUpdate(const Vector &S, bool quick); - - void SetH0(double h0) { quad_data.h0 = h0; } - double GetH0() const { return quad_data.h0; } - - Vector& GetZoneMaxVisc() { return zone_max_visc; } - Vector& GetZoneVGrad() { return zone_vgrad; } - - void PrintTimingData(bool IamRoot, int steps); - - ~LagrangianHydroOperator(); -}; - -class TaylorCoefficient : public Coefficient -{ - virtual double Eval(ElementTransformation &T, - const IntegrationPoint &ip) - { - Vector x(2); - T.Transform(ip, x); - return 3.0 / 8.0 * M_PI * ( cos(3.0*M_PI*x(0)) * cos(M_PI*x(1)) - - cos(M_PI*x(0)) * cos(3.0*M_PI*x(1)) ); - } -}; - -} // namespace hydrodynamics - -} // namespace mfem - -#endif // MFEM_USE_MPI - -#endif // MFEM_LAGHOS diff --git a/amr/makefile b/amr/makefile deleted file mode 100644 index 0a7fdd3a..00000000 --- a/amr/makefile +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at -# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights -# reserved. See files LICENSE and NOTICE for details. -# -# This file is part of CEED, a collection of benchmarks, miniapps, software -# libraries and APIs for efficient high-order finite element and spectral -# element discretizations for exascale applications. For more information and -# source code availability see http://github.com/ceed. -# -# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, -# a collaborative effort of two U.S. Department of Energy organizations (Office -# of Science and the National Nuclear Security Administration) responsible for -# the planning and preparation of a capable exascale ecosystem, including -# software, applications, hardware, advanced system engineering and early -# testbed platforms, in support of the nation's exascale computing imperative. - -define LAGHOS_HELP_MSG - -Laghos makefile targets: - - make - make status/info - make install - make clean - make distclean - make style - -Examples: - -make -j 4 - Build Laghos using the current configuration options from MFEM. - (Laghos requires the MFEM finite element library, and uses its compiler and - linker options in its build process.) -make status - Display information about the current configuration. -make install PREFIX= - Install the Laghos executable in . -make clean - Clean the Laghos executable, library and object files. -make distclean - In addition to "make clean", remove the local installation directory and some - run-time generated files. -make style - Format the Laghos C++ source files using the Artistic Style (astyle) settings - from MFEM. - -endef - -# Default installation location -PREFIX = ./bin -INSTALL = /usr/bin/install - -# Use the MFEM build directory -MFEM_DIR = ../../mfem -CONFIG_MK = $(MFEM_DIR)/config/config.mk -TEST_MK = $(MFEM_DIR)/config/test.mk -# Use the MFEM install directory -# MFEM_DIR = ../mfem/mfem -# CONFIG_MK = $(MFEM_DIR)/config.mk -# TEST_MK = $(MFEM_DIR)/test.mk - -# Use two relative paths to MFEM: first one for compilation in '.' and second -# one for compilation in 'lib'. -MFEM_DIR1 := $(MFEM_DIR) -MFEM_DIR2 := $(realpath $(MFEM_DIR)) - -# Use the compiler used by MFEM. Get the compiler and the options for compiling -# and linking from MFEM's config.mk. (Skip this if the target does not require -# building.) -MFEM_LIB_FILE = mfem_is_not_built -ifeq (,$(filter help clean distclean style,$(MAKECMDGOALS))) - -include $(CONFIG_MK) -endif - -CXX = $(MFEM_CXX) -CPPFLAGS = $(MFEM_CPPFLAGS) -CXXFLAGS = $(MFEM_CXXFLAGS) - -# MFEM config does not define C compiler -CC = gcc -CFLAGS = -O3 - -# Optional link flags -LDFLAGS = - -OPTIM_OPTS = -O3 -DEBUG_OPTS = -g -Wall -LAGHOS_DEBUG = $(MFEM_DEBUG) -ifneq ($(LAGHOS_DEBUG),$(MFEM_DEBUG)) - ifeq ($(LAGHOS_DEBUG),YES) - CXXFLAGS = $(DEBUG_OPTS) - else - CXXFLAGS = $(OPTIM_OPTS) - endif -endif - -LAGHOS_FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(MFEM_INCFLAGS) -LAGHOS_LIBS = $(MFEM_LIBS) - -ifeq ($(LAGHOS_DEBUG),YES) - LAGHOS_FLAGS += -DLAGHOS_DEBUG -endif - -LIBS = $(strip $(LAGHOS_LIBS) $(LDFLAGS)) -CCC = $(strip $(CXX) $(LAGHOS_FLAGS)) -Ccc = $(strip $(CC) $(CFLAGS) $(GL_OPTS)) - -SOURCE_FILES = laghos.cpp laghos_solver.cpp laghos_assembly.cpp -OBJECT_FILES1 = $(SOURCE_FILES:.cpp=.o) -OBJECT_FILES = $(OBJECT_FILES1:.c=.o) -HEADER_FILES = laghos_solver.hpp laghos_assembly.hpp - -# Targets - -.PHONY: all clean distclean install status info opt debug test style clean-build clean-exec - -.SUFFIXES: .c .cpp .o -.cpp.o: - cd $( Date: Wed, 13 Jan 2021 10:29:08 -0800 Subject: [PATCH 11/22] Cleanup --- laghos.cpp | 615 ++++++++++---------------------------------- laghos_assembly.cpp | 231 ++++++++++++++--- laghos_assembly.hpp | 16 +- laghos_solver.cpp | 327 ++++++----------------- laghos_solver.hpp | 393 +++++++++++++++++++++++++--- 5 files changed, 786 insertions(+), 796 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index bd177bcc..7dcf6b59 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -60,40 +60,26 @@ // -m data/cube_12_hex.mesh -pt 322 for 12 / 96 / 768 / 6144 ... tasks. #include +#include #include #include #include "laghos_solver.hpp" -using std::cout; -using std::endl; using namespace mfem; -// Choice for the problem setup. -static int problem; - -// Forward declarations. -double e0(const Vector &); -double rho0(const Vector &); -double gamma_func(const Vector &); -void v0(const Vector &, Vector &); - static long GetMaxRssMB(); -static void display_banner(std::ostream&); -static void Checks(const int dim, const int ti, const double norm, int &checks); - -static void AMRUpdate(BlockVector&, BlockVector&, Array&, - ParGridFunction&, ParGridFunction&, - ParGridFunction&, ParGridFunction&); - -static void GetZeroBCDofs(ParMesh*, ParFiniteElementSpace&, const int, - Array&, Array&); - -static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements); - -static void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order = -1); +static void Checks(const int, const int, const double, int&); +static void display_banner(std::ostream &os) +{ + os << std::endl + << " __ __ " << std::endl + << " / / ____ ____ / /_ ____ _____ " << std::endl + << " / / / __ `/ __ `/ __ \\/ __ \\/ ___/ " << std::endl + << " / /___/ /_/ / /_/ / / / / /_/ (__ ) " << std::endl + << " /_____/\\__,_/\\__, /_/ /_/\\____/____/ " << std::endl + << " /____/ " << std::endl + << std::endl; +} int main(int argc, char *argv[]) { @@ -102,7 +88,7 @@ int main(int argc, char *argv[]) const int myid = mpi.WorldRank(); // Print the banner. - if (mpi.Root()) { display_banner(cout); } + if (mpi.Root()) { display_banner(std::cout); } // Parse command-line options. problem = 1; @@ -230,19 +216,20 @@ int main(int argc, char *argv[]) args.Parse(); if (!args.Good()) { - if (mpi.Root()) { args.PrintUsage(cout); } + if (mpi.Root()) { args.PrintUsage(std::cout); } return 1; } amr_max_level = std::max(amr_max_level, rs_levels + rp_levels); - if (mpi.Root()) { args.PrintOptions(cout); } + if (mpi.Root()) { args.PrintOptions(std::cout); } - /*if (amr && problem != 1) + if (amr && problem != 1) { - if (mpi.Root()) { cout << "AMR only supported for problem 1." << endl; } + if (mpi.Root()) + { std::cout << "AMR only supported for problem 1." << std::endl; } return 0; - }*/ + } // Configure the device from the command line options Device backend; @@ -287,7 +274,7 @@ int main(int argc, char *argv[]) bel->SetAttribute(attr); } } - else { MFEM_ABORT("Dimension should be set"); } + else { MFEM_ABORT("Dimension should be set"); mesh = new Mesh(); } } dim = mesh->Dimension(); @@ -297,7 +284,8 @@ int main(int argc, char *argv[]) p_assembly = false; if (mpi.Root()) { - cout << "Laghos does not support PA in 1D. Switching to FA." << endl; + std::cout << "Laghos does not support PA in 1D. Switching to FA." + << std::endl; } } @@ -319,7 +307,7 @@ int main(int argc, char *argv[]) const int mesh_NE = mesh->GetNE(); if (mpi.Root()) { - cout << "Number of zones in the serial mesh: " << mesh_NE << endl; + std::cout << "Number of zones in the serial mesh: " << mesh_NE << std::endl; } // Parallel partitioning of the mesh. @@ -407,7 +395,7 @@ int main(int argc, char *argv[]) default: if (myid == 0) { - cout << "Unknown partition type: " << partition_type << '\n'; + std::cout << "Unknown partition type: " << partition_type << '\n'; } delete mesh; MPI_Finalize(); @@ -415,7 +403,7 @@ int main(int argc, char *argv[]) } int product = 1; for (int d = 0; d < dim; d++) { product *= nxyz[d]; } - const bool cartesian_partitioning = (cxyz.Size() > 0)?true:false; + const bool cartesian_partitioning = (cxyz.Size() > 0) ? true : false; if (product == num_tasks || cartesian_partitioning) { if (cartesian_partitioning) @@ -438,7 +426,7 @@ int main(int argc, char *argv[]) { if (myid == 0) { - cout << "Non-Cartesian partitioning through METIS will be used.\n"; + std::cout << "Non-Cartesian partitioning through METIS will be used.\n"; #ifndef MFEM_USE_METIS cout << "MFEM was built without METIS. " << "Adjust the number of tasks to use a Cartesian split." << endl; @@ -459,7 +447,7 @@ int main(int argc, char *argv[]) MPI_Reduce(&NE, &ne_min, 1, MPI_INT, MPI_MIN, 0, pmesh->GetComm()); MPI_Reduce(&NE, &ne_max, 1, MPI_INT, MPI_MAX, 0, pmesh->GetComm()); if (myid == 0) - { cout << "Zones min/max: " << ne_min << " " << ne_max << endl; } + { std::cout << "Zones min/max: " << ne_min << " " << ne_max << std::endl; } // Define the parallel finite element spaces. We use: // - H1 (Gauss-Lobatto, continuous) for position and velocity. @@ -488,7 +476,7 @@ int main(int argc, char *argv[]) default: if (myid == 0) { - cout << "Unknown ODE solver type: " << ode_solver_type << '\n'; + std::cout << "Unknown ODE solver type: " << ode_solver_type << '\n'; } delete pmesh; MPI_Finalize(); @@ -499,10 +487,10 @@ int main(int argc, char *argv[]) const HYPRE_Int glob_size_h1 = H1FESpace.GlobalTrueVSize(); if (mpi.Root()) { - cout << "Number of kinematic (position, velocity) dofs: " - << glob_size_h1 << endl; - cout << "Number of specific internal energy dofs: " - << glob_size_l2 << endl; + std::cout << "Number of kinematic (position, velocity) dofs: " + << glob_size_h1 << '\n'; + std::cout << "Number of specific internal energy dofs: " + << glob_size_l2 << '\n'; } // The monolithic BlockVector stores unknown fields as: @@ -659,27 +647,7 @@ int main(int argc, char *argv[]) BlockVector S_old(S); long mem = 0, mmax = 0, msum = 0; int checks = 0; - // const double internal_energy = hydro.InternalEnergy(e_gf); - // const double kinetic_energy = hydro.KineticEnergy(v_gf); - // if (mpi.Root()) - // { - // cout << std::fixed; - // cout << "step " << std::setw(5) << 0 - // << ",\tt = " << std::setw(5) << std::setprecision(4) << t - // << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt - // << ",\t|IE| = " << std::setprecision(10) << std::scientific - // << internal_energy - // << ",\t|KE| = " << std::setprecision(10) << std::scientific - // << kinetic_energy - // << ",\t|E| = " << std::setprecision(10) << std::scientific - // << kinetic_energy+internal_energy; - // cout << std::fixed; - // if (mem_usage) - // { - // cout << ", mem: " << mmax << "/" << msum << " MB"; - // } - // cout << endl; - // } + for (int ti = 1; !last_step; ti++) { if (t + dt >= t_final) @@ -709,7 +677,7 @@ int main(int argc, char *argv[]) t = t_old; S = S_old; hydro.ResetQuadratureData(); - if (mpi.Root()) { cout << "Repeating step " << ti << endl; } + if (mpi.Root()) { std::cout << "Repeating step " << ti << '\n'; } if (steps < max_tsteps) { last_step = false; } ti--; continue; } @@ -737,24 +705,22 @@ int main(int argc, char *argv[]) MPI_Reduce(&mem, &mmax, 1, MPI_LONG, MPI_MAX, 0, pmesh->GetComm()); MPI_Reduce(&mem, &msum, 1, MPI_LONG, MPI_SUM, 0, pmesh->GetComm()); } - // const double internal_energy = hydro.InternalEnergy(e_gf); - // const double kinetic_energy = hydro.KineticEnergy(v_gf); if (mpi.Root()) { const double sqrt_norm = sqrt(norm); - cout << std::fixed; - cout << "step " << std::setw(5) << ti - << ",\tt = " << std::setw(5) << std::setprecision(4) << t - << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt - << ",\t|e| = " << std::setprecision(10) << std::scientific - << sqrt_norm; - cout << std::fixed; + std::cout << std::fixed; + std::cout << "step " << std::setw(5) << ti + << ",\tt = " << std::setw(5) << std::setprecision(4) << t + << ",\tdt = " << std::setw(5) << std::setprecision(6) << dt + << ",\t|e| = " << std::setprecision(10) << std::scientific + << sqrt_norm; + std::cout << std::fixed; if (mem_usage) { - cout << ", mem: " << mmax << "/" << msum << " MB"; + std::cout << ", mem: " << mmax << "/" << msum << " MB"; } - cout << endl; + std::cout << std::endl; } // Make sure all ranks have sent their 'v' solution before initiating @@ -940,12 +906,12 @@ int main(int argc, char *argv[]) { constexpr bool quick = true; - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); + amr::Update(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); hydro.AMRUpdate(S, quick); pmesh->Rebalance(); - AMRUpdate(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); + amr::Update(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); hydro.AMRUpdate(S, !quick); GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); @@ -995,13 +961,13 @@ int main(int argc, char *argv[]) hydro.KineticEnergy(v_gf); if (mpi.Root()) { - cout << endl; - cout << "Energy diff: " << std::scientific << std::setprecision(2) - << fabs(energy_init - energy_final) << endl; + std::cout << std::endl; + std::cout << "Energy diff: " << std::scientific << std::setprecision(2) + << fabs(energy_init - energy_final) << std::endl; if (mem_usage) { - cout << "Maximum memory resident set size: " - << mmax << "/" << msum << " MB" << endl; + std::cout << "Maximum memory resident set size: " + << mmax << "/" << msum << " MB" << std::endl; } } @@ -1014,9 +980,9 @@ int main(int argc, char *argv[]) error_l2 = v_gf.ComputeL2Error(v_coeff); if (mpi.Root()) { - cout << "L_inf error: " << error_max << endl - << "L_1 error: " << error_l1 << endl - << "L_2 error: " << error_l2 << endl; + std::cout << "L_inf error: " << error_max << std::endl + << "L_1 error: " << error_l1 << std::endl + << "L_2 error: " << error_l2 << std::endl; } } @@ -1033,337 +999,7 @@ int main(int argc, char *argv[]) return 0; } -void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, - const int bdr_attr_max, - Array &ess_tdofs, - Array &ess_vdofs) -{ - ess_tdofs.SetSize(0); - ess_vdofs.SetSize(0); - Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; - for (int d = 0; d < pmesh->Dimension(); d++) - { - // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, - // i.e., we must enforce v_x/y/z = 0 for the velocity components. - ess_bdr = 0; ess_bdr[d] = 1; - H1.GetEssentialTrueDofs(ess_bdr, dofs_list, d); - ess_tdofs.Append(dofs_list); - H1.GetEssentialVDofs(ess_bdr, dofs_marker, d); - FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); - ess_vdofs.Append(dofs_list); - } -} - -void AMRUpdate(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf, - ParGridFunction &m_gf) -{ - ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); - ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); - - H1FESpace->Update(); - L2FESpace->Update(); - MEFESpace->Update(); - - const int Vsize_h1 = H1FESpace->GetVSize(); - const int Vsize_l2 = L2FESpace->GetVSize(); - - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - - S_tmp = S; - S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); - - x_gf.MakeRef(H1FESpace, S, true_offset[0]); - v_gf.MakeRef(H1FESpace, S, true_offset[1]); - e_gf.MakeRef(L2FESpace, S, true_offset[2]); - x_gf.SyncAliasMemory(S); - v_gf.SyncAliasMemory(S); - e_gf.SyncAliasMemory(S); - - S_tmp.Update(true_offset); - m_gf.Update(); -} - -void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements) -{ - Array v; - - for (int i = 0; i < mesh->GetNE(); i++) - { - mesh->GetElementVertices(i, v); - for (int j = 0; j < v.Size(); j++) - { - double dist = 0.0; - for (int l = 0; l < mesh->SpaceDimension(); l++) - { - double d = vert(l) - mesh->GetVertex(v[j])[l]; - dist += d*d; - } - if (dist <= size*size) { elements.Append(i); break; } - } - } -} - -static void Pow(Vector &vec, double p) -{ - for (int i = 0; i < vec.Size(); i++) - { - vec(i) = std::pow(vec(i), p); - } -} - -void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order) -{ - const FiniteElementSpace *space = gf.FESpace(); - int ne = space->GetNE(); - - if (int_order < 0) { int_order = space->GetOrder(0) + 1; } - - elem_min.SetSize(ne); - elem_max.SetSize(ne); - - Vector vals, tmp; - for (int i = 0; i < ne; i++) - { - int geom = space->GetFE(i)->GetGeomType(); - const IntegrationRule &ir = IntRules.Get(geom, int_order); - - gf.GetValues(i, ir, vals); - - if (space->GetVDim() > 1) - { - Pow(vals, 2.0); - for (int vd = 1; vd < space->GetVDim(); vd++) - { - gf.GetValues(i, ir, tmp, vd+1); - Pow(tmp, 2.0); - vals += tmp; - } - Pow(vals, 0.5); - } - - elem_min(i) = vals.Min(); - elem_max(i) = vals.Max(); - } -} - -double rho0(const Vector &x) -{ - switch (problem) - { - case 0: return 1.0; - case 1: return 1.0; - case 2: return (x(0) < 0.5) ? 1.0 : 0.1; - case 3: return (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0; - case 4: return 1.0; - case 5: - { - if (x(0) >= 0.5 && x(1) >= 0.5) { return 0.5313; } - if (x(0) < 0.5 && x(1) < 0.5) { return 0.8; } - return 1.0; - } - case 6: - { - if (x(0) < 0.5 && x(1) >= 0.5) { return 2.0; } - if (x(0) >= 0.5 && x(1) < 0.5) { return 3.0; } - return 1.0; - } - case 7: return x(1) >= 0.0 ? 2.0 : 1.0; - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -double gamma_func(const Vector &x) -{ - switch (problem) - { - case 0: return 5.0 / 3.0; - case 1: return 1.4; - case 2: return 1.4; - case 3: return (x(0) > 1.0 && x(1) <= 1.5) ? 1.4 : 1.5; - case 4: return 5.0 / 3.0; - case 5: return 1.4; - case 6: return 1.4; - case 7: return 5.0 / 3.0; - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -static double rad(double x, double y) { return sqrt(x*x + y*y); } - -void v0(const Vector &x, Vector &v) -{ - const double atn = pow((x(0)*(1.0-x(0))*4*x(1)*(1.0-x(1))*4.0),0.4); - switch (problem) - { - case 0: - v(0) = sin(M_PI*x(0)) * cos(M_PI*x(1)); - v(1) = -cos(M_PI*x(0)) * sin(M_PI*x(1)); - if (x.Size() == 3) - { - v(0) *= cos(M_PI*x(2)); - v(1) *= cos(M_PI*x(2)); - v(2) = 0.0; - } - break; - case 1: v = 0.0; break; - case 2: v = 0.0; break; - case 3: v = 0.0; break; - case 4: - { - v = 0.0; - const double r = rad(x(0), x(1)); - if (r < 0.2) - { - v(0) = 5.0 * x(1); - v(1) = -5.0 * x(0); - } - else if (r < 0.4) - { - v(0) = 2.0 * x(1) / r - 5.0 * x(1); - v(1) = -2.0 * x(0) / r + 5.0 * x(0); - } - else { } - break; - } - case 5: - { - v = 0.0; - if (x(0) >= 0.5 && x(1) >= 0.5) { v(0)=0.0*atn, v(1)=0.0*atn; return;} - if (x(0) < 0.5 && x(1) >= 0.5) { v(0)=0.7276*atn, v(1)=0.0*atn; return;} - if (x(0) < 0.5 && x(1) < 0.5) { v(0)=0.0*atn, v(1)=0.0*atn; return;} - if (x(0) >= 0.5 && x(1) < 0.5) { v(0)=0.0*atn, v(1)=0.7276*atn; return; } - MFEM_ABORT("Error in problem 5!"); - return; - } - case 6: - { - v = 0.0; - if (x(0) >= 0.5 && x(1) >= 0.5) { v(0)=+0.75*atn, v(1)=-0.5*atn; return;} - if (x(0) < 0.5 && x(1) >= 0.5) { v(0)=+0.75*atn, v(1)=+0.5*atn; return;} - if (x(0) < 0.5 && x(1) < 0.5) { v(0)=-0.75*atn, v(1)=+0.5*atn; return;} - if (x(0) >= 0.5 && x(1) < 0.5) { v(0)=-0.75*atn, v(1)=-0.5*atn; return;} - MFEM_ABORT("Error in problem 6!"); - return; - } - case 7: - { - v = 0.0; - v(1) = 0.02 * exp(-2*M_PI*x(1)*x(1)) * cos(2*M_PI*x(0)); - break; - } - default: MFEM_ABORT("Bad number given for problem id!"); - } -} - -double e0(const Vector &x) -{ - switch (problem) - { - case 0: - { - const double denom = 2.0 / 3.0; // (5/3 - 1) * density. - double val; - if (x.Size() == 2) - { - val = 1.0 + (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) / 4.0; - } - else - { - val = 100.0 + ((cos(2*M_PI*x(2)) + 2) * - (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) - 2) / 16.0; - } - return val/denom; - } - case 1: return 0.0; // This case in initialized in main(). - case 2: return (x(0) < 0.5) ? 1.0 / rho0(x) / (gamma_func(x) - 1.0) - : 0.1 / rho0(x) / (gamma_func(x) - 1.0); - case 3: return (x(0) > 1.0) ? 0.1 / rho0(x) / (gamma_func(x) - 1.0) - : 1.0 / rho0(x) / (gamma_func(x) - 1.0); - case 4: - { - const double r = rad(x(0), x(1)), rsq = x(0) * x(0) + x(1) * x(1); - const double gamma = 5.0 / 3.0; - if (r < 0.2) - { - return (5.0 + 25.0 / 2.0 * rsq) / (gamma - 1.0); - } - else if (r < 0.4) - { - const double t1 = 9.0 - 4.0 * log(0.2) + 25.0 / 2.0 * rsq; - const double t2 = 20.0 * r - 4.0 * log(r); - return (t1 - t2) / (gamma - 1.0); - } - else { return (3.0 + 4.0 * log(2.0)) / (gamma - 1.0); } - } - case 5: - { - const double irg = 1.0 / rho0(x) / (gamma_func(x) - 1.0); - if (x(0) >= 0.5 && x(1) >= 0.5) { return 0.4 * irg; } - if (x(0) < 0.5 && x(1) >= 0.5) { return 1.0 * irg; } - if (x(0) < 0.5 && x(1) < 0.5) { return 1.0 * irg; } - if (x(0) >= 0.5 && x(1) < 0.5) { return 1.0 * irg; } - MFEM_ABORT("Error in problem 5!"); - return 0.0; - } - case 6: - { - const double irg = 1.0 / rho0(x) / (gamma_func(x) - 1.0); - if (x(0) >= 0.5 && x(1) >= 0.5) { return 1.0 * irg; } - if (x(0) < 0.5 && x(1) >= 0.5) { return 1.0 * irg; } - if (x(0) < 0.5 && x(1) < 0.5) { return 1.0 * irg; } - if (x(0) >= 0.5 && x(1) < 0.5) { return 1.0 * irg; } - MFEM_ABORT("Error in problem 5!"); - return 0.0; - } - case 7: - { - const double rho = rho0(x), gamma = gamma_func(x); - return (6.0 - rho * x(1)) / (gamma - 1.0) / rho; - } - default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; - } -} - -static void display_banner(std::ostream &os) -{ - os << endl - << " __ __ " << endl - << " / / ____ ____ / /_ ____ _____ " << endl - << " / / / __ `/ __ `/ __ \\/ __ \\/ ___/ " << endl - << " / /___/ /_/ / /_/ / / / / /_/ (__ ) " << endl - << " /_____/\\__,_/\\__, /_/ /_/\\____/____/ " << endl - << " /____/ " << endl << endl; -} - -static long GetMaxRssMB() -{ - struct rusage usage; - if (getrusage(RUSAGE_SELF, &usage)) { return -1; } -#ifndef __APPLE__ - const long unit = 1024; // kilo -#else - const long unit = 1024*1024; // mega -#endif - return usage.ru_maxrss/unit; // mega bytes -} - -static bool rerr(const double a, const double v, const double eps) +static bool Check(const double a, const double v, const double eps) { MFEM_VERIFY(fabs(a) > eps && fabs(v) > eps, "One value is near zero!"); const double err_a = fabs((a-v)/a); @@ -1378,72 +1014,85 @@ static void Checks(const int dim, const int ti, const double nrm, int &chk) printf("%.15e\n",nrm); if (dim==2) { - const double p0_05 = 6.54653862453438e+00; - const double p0_27 = 7.58857635779292e+00; - if (pb==0 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p0_05,eps),"P0, #05");} - if (pb==0 && ti==27) {chk++; MFEM_VERIFY(rerr(nrm,p0_27,eps),"P0, #27");} - const double p1_05 = 3.50825494522579e+00; - const double p1_15 = 2.75644459682321e+00; - if (pb==1 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p1_05,eps),"P1, #05");} - if (pb==1 && ti==15) {chk++; MFEM_VERIFY(rerr(nrm,p1_15,eps),"P1, #15");} - const double p2_05 = 1.02074579565124e+01; - const double p2_59 = 1.72159020590190e+01; - if (pb==2 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p2_05,eps),"P2, #05");} - if (pb==2 && ti==59) {chk++; MFEM_VERIFY(rerr(nrm,p2_59,eps),"P2, #59");} - const double p3_05 = 8.0; - const double p3_16 = 8.0; - if (pb==3 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p3_05,eps),"P3, #05");} - if (pb==3 && ti==16) {chk++; MFEM_VERIFY(rerr(nrm,p3_16,eps),"P3, #16");} - const double p4_05 = 3.446324942352448e+01; - const double p4_18 = 3.446844033767240e+01; - if (pb==4 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p4_05,eps),"P4, #05");} - if (pb==4 && ti==18) {chk++; MFEM_VERIFY(rerr(nrm,p4_18,eps),"P4, #18");} - const double p5_05 = 1.030899557252528e+01; - const double p5_36 = 1.057362418574309e+01; - if (pb==5 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p5_05,eps),"P5, #05");} - if (pb==5 && ti==36) {chk++; MFEM_VERIFY(rerr(nrm,p5_36,eps),"P5, #36");} - const double p6_05 = 8.039707010835693e+00; - const double p6_36 = 8.316970976817373e+00; - if (pb==6 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p6_05,eps),"P6, #05");} - if (pb==6 && ti==36) {chk++; MFEM_VERIFY(rerr(nrm,p6_36,eps),"P6, #36");} - const double p7_05 = 1.514929259650760e+01; - const double p7_25 = 1.514931278155159e+01; - if (pb==7 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p7_05,eps),"P7, #05");} - if (pb==7 && ti==25) {chk++; MFEM_VERIFY(rerr(nrm,p7_25,eps),"P7, #25");} + constexpr double p0_05 = 6.54653862453438e+00; + constexpr double p0_27 = 7.58857635779292e+00; + if (pb==0 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p0_05,eps),"P0, #05");} + if (pb==0 && ti==27) {chk++; MFEM_VERIFY(Check(nrm,p0_27,eps),"P0, #27");} + constexpr double p1_05 = 3.50825494522579e+00; + constexpr double p1_15 = 2.75644459682321e+00; + if (pb==1 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p1_05,eps),"P1, #05");} + if (pb==1 && ti==15) {chk++; MFEM_VERIFY(Check(nrm,p1_15,eps),"P1, #15");} + constexpr double p2_05 = 1.02074579565124e+01; + constexpr double p2_59 = 1.72159020590190e+01; + if (pb==2 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p2_05,eps),"P2, #05");} + if (pb==2 && ti==59) {chk++; MFEM_VERIFY(Check(nrm,p2_59,eps),"P2, #59");} + constexpr double p3_05 = 8.0; + constexpr double p3_16 = 8.0; + if (pb==3 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p3_05,eps),"P3, #05");} + if (pb==3 && ti==16) {chk++; MFEM_VERIFY(Check(nrm,p3_16,eps),"P3, #16");} + constexpr double p4_05 = 3.446324942352448e+01; + constexpr double p4_18 = 3.446844033767240e+01; + if (pb==4 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p4_05,eps),"P4, #05");} + if (pb==4 && ti==18) {chk++; MFEM_VERIFY(Check(nrm,p4_18,eps),"P4, #18");} + constexpr double p5_05 = 1.030899557252528e+01; + constexpr double p5_36 = 1.057362418574309e+01; + if (pb==5 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p5_05,eps),"P5, #05");} + if (pb==5 && ti==36) {chk++; MFEM_VERIFY(Check(nrm,p5_36,eps),"P5, #36");} + constexpr double p6_05 = 8.039707010835693e+00; + constexpr double p6_36 = 8.316970976817373e+00; + if (pb==6 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p6_05,eps),"P6, #05");} + if (pb==6 && ti==36) {chk++; MFEM_VERIFY(Check(nrm,p6_36,eps),"P6, #36");} + constexpr double p7_05 = 1.514929259650760e+01; + constexpr double p7_25 = 1.514931278155159e+01; + if (pb==7 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p7_05,eps),"P7, #05");} + if (pb==7 && ti==25) {chk++; MFEM_VERIFY(Check(nrm,p7_25,eps),"P7, #25");} } if (dim==3) { - const double p0_05 = 1.198510951452527e+03; - const double p0_188 = 1.199384410059154e+03; - if (pb==0 && ti==005) {chk++; MFEM_VERIFY(rerr(nrm,p0_05,eps),"P0, #05");} - if (pb==0 && ti==188) {chk++; MFEM_VERIFY(rerr(nrm,p0_188,eps),"P0, #188");} - const double p1_05 = 1.33916371859257e+01; - const double p1_28 = 7.52107367739800e+00; - if (pb==1 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p1_05,eps),"P1, #05");} - if (pb==1 && ti==28) {chk++; MFEM_VERIFY(rerr(nrm,p1_28,eps),"P1, #28");} - const double p2_05 = 2.041491591302486e+01; - const double p2_59 = 3.443180411803796e+01; - if (pb==2 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p2_05,eps),"P2, #05");} - if (pb==2 && ti==59) {chk++; MFEM_VERIFY(rerr(nrm,p2_59,eps),"P2, #59");} - const double p3_05 = 1.600000000000000e+01; - const double p3_16 = 1.600000000000000e+01; - if (pb==3 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p3_05,eps),"P3, #05");} - if (pb==3 && ti==16) {chk++; MFEM_VERIFY(rerr(nrm,p3_16,eps),"P3, #16");} - const double p4_05 = 6.892649884704898e+01; - const double p4_18 = 6.893688067534482e+01; - if (pb==4 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p4_05,eps),"P4, #05");} - if (pb==4 && ti==18) {chk++; MFEM_VERIFY(rerr(nrm,p4_18,eps),"P4, #18");} - const double p5_05 = 2.061984481890964e+01; - const double p5_36 = 2.114519664792607e+01; - if (pb==5 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p5_05,eps),"P5, #05");} - if (pb==5 && ti==36) {chk++; MFEM_VERIFY(rerr(nrm,p5_36,eps),"P5, #36");} - const double p6_05 = 1.607988713996459e+01; - const double p6_36 = 1.662736010353023e+01; - if (pb==6 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p6_05,eps),"P6, #05");} - if (pb==6 && ti==36) {chk++; MFEM_VERIFY(rerr(nrm,p6_36,eps),"P6, #36");} - const double p7_05 = 3.029858112572883e+01; - const double p7_24 = 3.029858832743707e+01; - if (pb==7 && ti==05) {chk++; MFEM_VERIFY(rerr(nrm,p7_05,eps),"P7, #05");} - if (pb==7 && ti==24) {chk++; MFEM_VERIFY(rerr(nrm,p7_24,eps),"P7, #24");} + constexpr double p0_05 = 1.198510951452527e+03; + constexpr double p0_188 = 1.199384410059154e+03; + if (pb==0 && ti==005) {chk++; MFEM_VERIFY(Check(nrm,p0_05,eps),"P0, #05");} + if (pb==0 && ti==188) {chk++; MFEM_VERIFY(Check(nrm,p0_188,eps),"P0, #188");} + constexpr double p1_05 = 1.33916371859257e+01; + constexpr double p1_28 = 7.52107367739800e+00; + if (pb==1 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p1_05,eps),"P1, #05");} + if (pb==1 && ti==28) {chk++; MFEM_VERIFY(Check(nrm,p1_28,eps),"P1, #28");} + constexpr double p2_05 = 2.041491591302486e+01; + constexpr double p2_59 = 3.443180411803796e+01; + if (pb==2 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p2_05,eps),"P2, #05");} + if (pb==2 && ti==59) {chk++; MFEM_VERIFY(Check(nrm,p2_59,eps),"P2, #59");} + constexpr double p3_05 = 1.600000000000000e+01; + constexpr double p3_16 = 1.600000000000000e+01; + if (pb==3 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p3_05,eps),"P3, #05");} + if (pb==3 && ti==16) {chk++; MFEM_VERIFY(Check(nrm,p3_16,eps),"P3, #16");} + constexpr double p4_05 = 6.892649884704898e+01; + constexpr double p4_18 = 6.893688067534482e+01; + if (pb==4 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p4_05,eps),"P4, #05");} + if (pb==4 && ti==18) {chk++; MFEM_VERIFY(Check(nrm,p4_18,eps),"P4, #18");} + constexpr double p5_05 = 2.061984481890964e+01; + constexpr double p5_36 = 2.114519664792607e+01; + if (pb==5 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p5_05,eps),"P5, #05");} + if (pb==5 && ti==36) {chk++; MFEM_VERIFY(Check(nrm,p5_36,eps),"P5, #36");} + constexpr double p6_05 = 1.607988713996459e+01; + constexpr double p6_36 = 1.662736010353023e+01; + if (pb==6 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p6_05,eps),"P6, #05");} + if (pb==6 && ti==36) {chk++; MFEM_VERIFY(Check(nrm,p6_36,eps),"P6, #36");} + constexpr double p7_05 = 3.029858112572883e+01; + constexpr double p7_24 = 3.029858832743707e+01; + if (pb==7 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p7_05,eps),"P7, #05");} + if (pb==7 && ti==24) {chk++; MFEM_VERIFY(Check(nrm,p7_24,eps),"P7, #24");} } } + +// Get current process information: max resident set size +static long GetMaxRssMB() +{ + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) { return -1; } +#ifndef __APPLE__ + constexpr long unit = 1024; +#else + constexpr long unit = 1024*1024; +#endif + return usage.ru_maxrss / unit; +} diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index 60ee07f3..4edc6124 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -14,8 +14,11 @@ // software, applications, hardware, advanced system engineering and early // testbed platforms, in support of the nation's exascale computing imperative. -#include "laghos_assembly.hpp" #include +#include "laghos_assembly.hpp" +#include "general/forall.hpp" + +using namespace mfem; namespace mfem { @@ -23,6 +26,170 @@ namespace mfem namespace hydrodynamics { +double ComputeVolumeIntegral(const int DIM, const int NE, const int NQ, + const int Q1D, const int VDIM, + const double ln_norm, + const Vector& mass, + const Vector& f) +{ + + Vector integrand(NE*NQ); + const auto f_vals = Reshape(f.Read(), VDIM, NQ, NE); + auto I = Reshape(integrand.Write(), NQ, NE); + + if (DIM == 1) + { + for (int e=0; e < NE; ++e) + { + for (int q = 0; q < NQ; ++q) + { + double vmag = 0; + for (int k = 0; k < VDIM; k++) + { + vmag += pow(f_vals(k,q,e),ln_norm); + } + I(q,e) = vmag; + } + } + } + else if (DIM == 2) + { + MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const int q = qx + qy * Q1D; + double vmag = 0; + for (int k = 0; k < VDIM; k++) + { + vmag += pow(f_vals(k,q,e),ln_norm); + } + I(q,e) = vmag; + } + } + }); + } + else if (DIM == 3) + { + MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, + { + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const int q = qx + (qy + qz * Q1D) * Q1D; + double vmag = 0; + for (int k = 0; k < VDIM; k++) + { + vmag += pow(f_vals(k,q,e),ln_norm); + } + I(q,e) = vmag; + } + } + } + }); + } + const double integral = integrand * mass; + return integral; +} + +void Rho0DetJ0Vol(const int dim, const int NE, + const IntegrationRule &ir, + ParMesh *pmesh, + ParFiniteElementSpace &L2, + const ParGridFunction &rho0, + hydrodynamics::QuadratureData &qdata, + double &volume) +{ + const int NQ = ir.GetNPoints(); + const int Q1D = IntRules.Get(Geometry::SEGMENT,ir.GetOrder()).GetNPoints(); + const int flags = GeometricFactors::JACOBIANS|GeometricFactors::DETERMINANTS; + const GeometricFactors *geom = pmesh->GetGeometricFactors(ir, flags); + Vector rho0Q(NQ*NE); + rho0Q.UseDevice(true); + Vector j, detj; + const QuadratureInterpolator *qi = L2.GetQuadratureInterpolator(ir); + qi->Mult(rho0, QuadratureInterpolator::VALUES, rho0Q, j, detj); + const auto W = ir.GetWeights().Read(); + const auto R = Reshape(rho0Q.Read(), NQ, NE); + const auto J = Reshape(geom->J.Read(), NQ, dim, dim, NE); + const auto detJ = Reshape(geom->detJ.Read(), NQ, NE); + auto V = Reshape(qdata.rho0DetJ0w.Write(), NQ, NE); + Memory &Jinv_m = qdata.Jac0inv.GetMemory(); + const MemoryClass mc = Device::GetMemoryClass(); + const int Ji_total_size = qdata.Jac0inv.TotalSize(); + auto invJ = Reshape(Jinv_m.Write(mc, Ji_total_size), dim, dim, NQ, NE); + Vector vol(NE*NQ), one(NE*NQ); + auto A = Reshape(vol.Write(), NQ, NE); + auto O = Reshape(one.Write(), NQ, NE); + MFEM_ASSERT(dim==2 || dim==3, ""); + if (dim==2) + { + MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const int q = qx + qy * Q1D; + const double J11 = J(q,0,0,e); + const double J12 = J(q,1,0,e); + const double J21 = J(q,0,1,e); + const double J22 = J(q,1,1,e); + const double det = detJ(q,e); + V(q,e) = W[q] * R(q,e) * det; + const double r_idetJ = 1.0 / det; + invJ(0,0,q,e) = J22 * r_idetJ; + invJ(1,0,q,e) = -J12 * r_idetJ; + invJ(0,1,q,e) = -J21 * r_idetJ; + invJ(1,1,q,e) = J11 * r_idetJ; + A(q,e) = W[q] * det; + O(q,e) = 1.0; + } + } + }); + } + else + { + MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, + { + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const int q = qx + (qy + qz * Q1D) * Q1D; + const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e); + const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e); + const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e); + const double det = detJ(q,e); + V(q,e) = W[q] * R(q,e) * det; + const double r_idetJ = 1.0 / det; + invJ(0,0,q,e) = r_idetJ * ((J22 * J33)-(J23 * J32)); + invJ(1,0,q,e) = r_idetJ * ((J32 * J13)-(J33 * J12)); + invJ(2,0,q,e) = r_idetJ * ((J12 * J23)-(J13 * J22)); + invJ(0,1,q,e) = r_idetJ * ((J23 * J31)-(J21 * J33)); + invJ(1,1,q,e) = r_idetJ * ((J33 * J11)-(J31 * J13)); + invJ(2,1,q,e) = r_idetJ * ((J13 * J21)-(J11 * J23)); + invJ(0,2,q,e) = r_idetJ * ((J21 * J32)-(J22 * J31)); + invJ(1,2,q,e) = r_idetJ * ((J31 * J12)-(J32 * J11)); + invJ(2,2,q,e) = r_idetJ * ((J11 * J22)-(J12 * J21)); + A(q,e) = W[q] * det; + O(q,e) = 1.0; + } + } + } + }); + } + qdata.rho0DetJ0w.HostRead(); + volume = vol * one; +} + void DensityIntegrator::AssembleRHSElementVect(const FiniteElement &fe, ElementTransformation &Tr, Vector &elvect) @@ -150,14 +317,16 @@ void ForceMult2D(const int NE, const DenseTensor &sJit_, const Vector &x, Vector &y) { - auto b = Reshape(B_.Read(), Q1D, L1D); - auto bt = Reshape(Bt_.Read(), D1D, Q1D); - auto gt = Reshape(Gt_.Read(), D1D, Q1D); - const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*NE*DIM*DIM); - auto sJit = Reshape(StressJinvT, Q1D, Q1D, NE, DIM, DIM); - auto energy = Reshape(x.Read(), L1D, L1D, NE); const double eps1 = std::numeric_limits::epsilon(); const double eps2 = eps1*eps1; + + const auto b = Reshape(B_.Read(), Q1D, L1D); + const auto bt = Reshape(Bt_.Read(), D1D, Q1D); + const auto gt = Reshape(Gt_.Read(), D1D, Q1D); + const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*NE*DIM*DIM); + const auto sJit = Reshape(StressJinvT, Q1D, Q1D, NE, DIM, DIM); + const auto energy = Reshape(x.Read(), L1D, L1D, NE); + auto velocity = Reshape(y.Write(), D1D, D1D, DIM, NE); MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, @@ -301,14 +470,16 @@ void ForceMult3D(const int NE, const DenseTensor &sJit_, const Vector &x, Vector &y) { - auto b = Reshape(B_.Read(), Q1D, L1D); - auto bt = Reshape(Bt_.Read(), D1D, Q1D); - auto gt = Reshape(Gt_.Read(), D1D, Q1D); - const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*Q1D*NE*DIM*DIM); - auto sJit = Reshape(StressJinvT, Q1D, Q1D, Q1D, NE, DIM, DIM); - auto energy = Reshape(x.Read(), L1D, L1D, L1D, NE); const double eps1 = std::numeric_limits::epsilon(); const double eps2 = eps1*eps1; + + const auto b = Reshape(B_.Read(), Q1D, L1D); + const auto bt = Reshape(Bt_.Read(), D1D, Q1D); + const auto gt = Reshape(Gt_.Read(), D1D, Q1D); + const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*Q1D*NE*DIM*DIM); + const auto sJit = Reshape(StressJinvT, Q1D, Q1D, Q1D, NE, DIM, DIM); + const auto energy = Reshape(x.Read(), L1D, L1D, L1D, NE); + auto velocity = Reshape(y.Write(), D1D, D1D, D1D, DIM, NE); MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, @@ -535,13 +706,13 @@ static void ForceMult(const int DIM, const int D1D, const int Q1D, static std::unordered_map call = { // 2D - {0x234,&ForceMult2D<2,3,4,2>}, - {0x246,&ForceMult2D<2,4,6,3>}, - {0x258,&ForceMult2D<2,5,8,4>}, + {0x234, &ForceMult2D<2,3,4,2>}, + {0x246, &ForceMult2D<2,4,6,3>}, + {0x258, &ForceMult2D<2,5,8,4>}, // 3D - {0x334,&ForceMult3D<3,3,4,2>}, - {0x346,&ForceMult3D<3,4,6,3>}, - {0x358,&ForceMult3D<3,5,8,4>}, + {0x334, &ForceMult3D<3,3,4,2>}, + {0x346, &ForceMult3D<3,4,6,3>}, + {0x358, &ForceMult3D<3,5,8,4>}, }; if (!call[id]) { @@ -569,12 +740,13 @@ void ForceMultTranspose2D(const int NE, const DenseTensor &sJit_, const Vector &x, Vector &y) { - auto b = Reshape(B_.Read(), Q1D, D1D); - auto g = Reshape(G_.Read(), Q1D, D1D); - auto bt = Reshape(Bt_.Read(), L1D, Q1D); + const auto b = Reshape(B_.Read(), Q1D, D1D); + const auto g = Reshape(G_.Read(), Q1D, D1D); + const auto bt = Reshape(Bt_.Read(), L1D, Q1D); const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*NE*DIM*DIM); - auto sJit = Reshape(StressJinvT, Q1D, Q1D, NE, DIM, DIM); - auto velocity = Reshape(x.Read(), D1D, D1D, DIM, NE); + const auto sJit = Reshape(StressJinvT, Q1D, Q1D, NE, DIM, DIM); + const auto velocity = Reshape(x.Read(), D1D, D1D, DIM, NE); + auto energy = Reshape(y.Write(), L1D, L1D, NE); MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ, @@ -718,12 +890,13 @@ void ForceMultTranspose3D(const int NE, const Vector &v_, Vector &e_) { - auto b = Reshape(B_.Read(), Q1D, D1D); - auto g = Reshape(G_.Read(), Q1D, D1D); - auto bt = Reshape(Bt_.Read(), L1D, Q1D); + const auto b = Reshape(B_.Read(), Q1D, D1D); + const auto g = Reshape(G_.Read(), Q1D, D1D); + const auto bt = Reshape(Bt_.Read(), L1D, Q1D); const double *StressJinvT = Read(sJit_.GetMemory(), Q1D*Q1D*Q1D*NE*DIM*DIM); - auto sJit = Reshape(StressJinvT, Q1D, Q1D, Q1D, NE, DIM, DIM); - auto velocity = Reshape(v_.Read(), D1D, D1D, D1D, DIM, NE); + const auto sJit = Reshape(StressJinvT, Q1D, Q1D, Q1D, NE, DIM, DIM); + const auto velocity = Reshape(v_.Read(), D1D, D1D, D1D, DIM, NE); + auto energy = Reshape(e_.Write(), L1D, L1D, L1D, NE); MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, diff --git a/laghos_assembly.hpp b/laghos_assembly.hpp index cb983e42..ba577d52 100644 --- a/laghos_assembly.hpp +++ b/laghos_assembly.hpp @@ -18,8 +18,6 @@ #define MFEM_LAGHOS_ASSEMBLY #include "mfem.hpp" -#include "general/forall.hpp" -#include "linalg/dtensor.hpp" namespace mfem { @@ -68,6 +66,20 @@ struct QuadratureData } }; +double ComputeVolumeIntegral(const int DIM, const int NE, const int NQ, + const int Q1D, const int VDIM, + const double ln_norm, + const mfem::Vector& mass, + const mfem::Vector& f); + +void Rho0DetJ0Vol(const int dim, const int NE, + const IntegrationRule &ir, + ParMesh *pmesh, + ParFiniteElementSpace &L2, + const ParGridFunction &rho0, + QuadratureData &qdata, + double &volume); + // This class is used only for visualization. It assembles (rho, phi) in each // zone, which is used by LagrangianHydroOperator::ComputeDensity to do an L2 // projection of the density. diff --git a/laghos_solver.cpp b/laghos_solver.cpp index 412e1c21..a81b9682 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -14,10 +14,11 @@ // software, applications, hardware, advanced system engineering and early // testbed platforms, in support of the nation's exascale computing imperative. -#include "general/forall.hpp" +#include +#include #include "laghos_solver.hpp" +#include "general/forall.hpp" #include "linalg/kernels.hpp" -#include #ifdef MFEM_USE_MPI @@ -80,14 +81,6 @@ void VisualizeField(socketstream &sock, const char *vishost, int visport, while (connection_failed); } -static void Rho0DetJ0Vol(const int dim, const int NE, - const IntegrationRule &ir, - ParMesh *pmesh, - ParFiniteElementSpace &L2, - const ParGridFunction &rho0, - QuadratureData &qdata, - double &volume); - LagrangianHydroOperator::LagrangianHydroOperator(const int size, ParFiniteElementSpace &h1, ParFiniteElementSpace &l2, @@ -689,113 +682,55 @@ void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) const } - -double ComputeVolumeIntegral(const int DIM, const int NE,const int NQ,const int Q1D,const int VDIM,const double ln_norm, -const mfem::Vector& mass, const mfem::Vector& f) -{ - - auto f_vals = mfem::Reshape(f.Read(),VDIM,NQ, NE); - mfem::Vector integrand(NE*NQ); - auto I = Reshape(integrand.Write(), NQ, NE); - - if (DIM == 1){ - for (int e=0; e < NE; ++e){ - for (int q = 0; q < NQ; ++q) { - double vmag = 0; - for(int k = 0; k < VDIM; k++){ - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - } - else if (DIM == 2){ - MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + qy * Q1D; - double vmag = 0; - for(int k = 0; k < VDIM; k++){ - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - }); - } - else if (DIM == 3){ - MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, - { - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + (qy + qz * Q1D) * Q1D; - double vmag = 0; - for(int k = 0; k < VDIM; k++){ - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - } - }); - -} - const double integral = integrand * mass; -return integral; - -} double LagrangianHydroOperator::InternalEnergy(const ParGridFunction &gf) const { double glob_ie = 0.0; - - // get the restriction and interpolator objects - const QuadratureInterpolator* l2_interpolator = L2.GetQuadratureInterpolator(ir); - l2_interpolator->SetOutputLayout(QVectorLayout::byVDIM); - auto L2r = L2.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); - const int NQ = ir.GetNPoints(); - const int ND = L2.GetFE(0)->GetDof(); + + // get the restriction and interpolator objects + const QuadratureInterpolator* l2_interpolator = L2.GetQuadratureInterpolator( + ir); + l2_interpolator->SetOutputLayout(QVectorLayout::byVDIM); + auto L2r = L2.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); + const int NQ = ir.GetNPoints(); + const int ND = L2.GetFE(0)->GetDof(); Vector e_vector(NE*ND), eintQ(NE*NQ); - // Get internal energy at the quadrature points - L2r->Mult(gf, e_vector); - l2_interpolator->Values(e_vector, eintQ); + // Get internal energy at the quadrature points + L2r->Mult(gf, e_vector); + l2_interpolator->Values(e_vector, eintQ); + + const double internal_energy = + ComputeVolumeIntegral(dim, NE, NQ, Q1D, 1, 1.0, qdata.rho0DetJ0w,eintQ); - double internal_energy = ComputeVolumeIntegral(dim,NE,NQ,Q1D,1,1.0,qdata.rho0DetJ0w,eintQ); - - MPI_Allreduce(&internal_energy, &glob_ie, 1, MPI_DOUBLE, MPI_SUM, L2.GetParMesh()->GetComm()); + MPI_Allreduce(&internal_energy, &glob_ie, 1, MPI_DOUBLE, MPI_SUM, + L2.GetParMesh()->GetComm()); return glob_ie; } double LagrangianHydroOperator::KineticEnergy(const ParGridFunction &v) const { - double glob_ke = 0.0; - - // get the restriction and interpolator objects - const QuadratureInterpolator* h1_interpolator = H1.GetQuadratureInterpolator(ir); - h1_interpolator->SetOutputLayout(QVectorLayout::byVDIM); - auto H1r = H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); - const int NQ = ir.GetNPoints(); - const int ND = H1.GetFE(0)->GetDof(); - Vector e_vector(dim*NE*ND), ekinQ(dim*NE*NQ); - - // Get internal energy at the quadrature points - H1r->Mult(v, e_vector); - h1_interpolator->Values(e_vector, ekinQ); + double glob_ke = 0.0; - // Get the IE, initial weighted mass + // get the restriction and interpolator objects + const QuadratureInterpolator* h1_interpolator = H1.GetQuadratureInterpolator( + ir); + h1_interpolator->SetOutputLayout(QVectorLayout::byVDIM); + auto H1r = H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC); + const int NQ = ir.GetNPoints(); + const int ND = H1.GetFE(0)->GetDof(); + Vector e_vector(dim*NE*ND), ekinQ(dim*NE*NQ); - double kinetic_energy = ComputeVolumeIntegral(dim,NE,NQ,Q1D,dim,2.0,qdata.rho0DetJ0w,ekinQ); - - MPI_Allreduce(&kinetic_energy, &glob_ke, 1, MPI_DOUBLE, MPI_SUM, H1.GetParMesh()->GetComm()); + // Get internal energy at the quadrature points + H1r->Mult(v, e_vector); + h1_interpolator->Values(e_vector, ekinQ); + // Get the IE, initial weighted mass + double kinetic_energy = + hydrodynamics::ComputeVolumeIntegral(dim, NE, NQ, Q1D, dim, 2.0, + qdata.rho0DetJ0w, ekinQ); + MPI_Allreduce(&kinetic_energy, &glob_ke, 1, MPI_DOUBLE, MPI_SUM, + H1.GetParMesh()->GetComm()); return 0.5*glob_ke; } @@ -819,7 +754,6 @@ void LagrangianHydroOperator::PrintTimingData(bool IamRoot, int steps, if (IamRoot) { - using namespace std; // FOM = (FOM1 * T1 + FOM2 * T2 + FOM3 * T3) / (T1 + T2 + T3) const HYPRE_Int H1iter = p_assembly ? (timer.H1iter/dim) : timer.H1iter; const double FOM1 = 1e-6 * H1GTVSize * H1iter / T[0]; @@ -827,57 +761,57 @@ void LagrangianHydroOperator::PrintTimingData(bool IamRoot, int steps, const double FOM3 = 1e-6 * alldata[1] * ir.GetNPoints() / T[3]; const double FOM = (FOM1 * T[0] + FOM2 * T[2] + FOM3 * T[3]) / T[4]; const double FOM0 = 1e-6 * steps * (H1GTVSize + L2GTVSize) / T[4]; - cout << endl; - cout << "CG (H1) total time: " << T[0] << endl; - cout << "CG (H1) rate (megadofs x cg_iterations / second): " - << FOM1 << endl; - cout << endl; - cout << "CG (L2) total time: " << T[1] << endl; - cout << "CG (L2) rate (megadofs x cg_iterations / second): " - << 1e-6 * alldata[0] / T[1] << endl; - cout << endl; - cout << "Forces total time: " << T[2] << endl; - cout << "Forces rate (megadofs x timesteps / second): " - << FOM2 << endl; - cout << endl; - cout << "UpdateQuadData total time: " << T[3] << endl; - cout << "UpdateQuadData rate (megaquads x timesteps / second): " - << FOM3 << endl; - cout << endl; - cout << "Major kernels total time (seconds): " << T[4] << endl; - cout << "Major kernels total rate (megadofs x time steps / second): " - << FOM << endl; + std::cout << std::endl; + std::cout << "CG (H1) total time: " << T[0] << std::endl; + std::cout << "CG (H1) rate (megadofs x cg_iterations / second): " + << FOM1 << std::endl; + std::cout << std::endl; + std::cout << "CG (L2) total time: " << T[1] << std::endl; + std::cout << "CG (L2) rate (megadofs x cg_iterations / second): " + << 1e-6 * alldata[0] / T[1] << std::endl; + std::cout << std::endl; + std::cout << "Forces total time: " << T[2] << std::endl; + std::cout << "Forces rate (megadofs x timesteps / second): " + << FOM2 << std::endl; + std::cout << std::endl; + std::cout << "UpdateQuadData total time: " << T[3] << std::endl; + std::cout << "UpdateQuadData rate (megaquads x timesteps / second): " + << FOM3 << std::endl; + std::cout << std::endl; + std::cout << "Major kernels total time (seconds): " << T[4] << std::endl; + std::cout << "Major kernels total rate (megadofs x time steps / second): " + << FOM << std::endl; if (!fom) { return; } const int QPT = ir.GetNPoints(); const HYPRE_Int GNZones = alldata[2]; const long ndofs = 2*H1GTVSize + L2GTVSize + QPT*GNZones; - cout << endl; - cout << "| Ranks " << "| Zones " - << "| H1 dofs " << "| L2 dofs " - << "| QP " << "| N dofs " - << "| FOM0 " - << "| FOM1 " << "| T1 " - << "| FOM2 " << "| T2 " - << "| FOM3 " << "| T3 " - << "| FOM " << "| TT " - << "|" << endl; - cout << setprecision(3); - cout << "| " << setw(6) << H1.GetNRanks() - << "| " << setw(8) << GNZones - << "| " << setw(8) << H1GTVSize - << "| " << setw(8) << L2GTVSize - << "| " << setw(3) << QPT - << "| " << setw(9) << ndofs - << "| " << setw(7) << FOM0 - << "| " << setw(7) << FOM1 - << "| " << setw(5) << T[0] - << "| " << setw(7) << FOM2 - << "| " << setw(5) << T[2] - << "| " << setw(7) << FOM3 - << "| " << setw(5) << T[3] - << "| " << setw(7) << FOM - << "| " << setw(5) << T[4] - << "| " << endl; + std::cout << std::endl; + std::cout << "| Ranks " << "| Zones " + << "| H1 dofs " << "| L2 dofs " + << "| QP " << "| N dofs " + << "| FOM0 " + << "| FOM1 " << "| T1 " + << "| FOM2 " << "| T2 " + << "| FOM3 " << "| T3 " + << "| FOM " << "| TT " + << "|" << std::endl; + std::cout << std::setprecision(3); + std::cout << "| " << std::setw(6) << H1.GetNRanks() + << "| " << std::setw(8) << GNZones + << "| " << std::setw(8) << H1GTVSize + << "| " << std::setw(8) << L2GTVSize + << "| " << std::setw(3) << QPT + << "| " << std::setw(9) << ndofs + << "| " << std::setw(7) << FOM0 + << "| " << std::setw(7) << FOM1 + << "| " << std::setw(5) << T[0] + << "| " << std::setw(7) << FOM2 + << "| " << std::setw(5) << T[2] + << "| " << std::setw(7) << FOM3 + << "| " << std::setw(5) << T[3] + << "| " << std::setw(7) << FOM + << "| " << std::setw(5) << T[4] + << "| " << std::endl; } } @@ -1270,99 +1204,6 @@ void QUpdateBody(const int NE, const int e, d_el_max_vgrad[e] = fmax(fabs(det_v_grad), d_el_max_vgrad[e]); } -static void Rho0DetJ0Vol(const int dim, const int NE, - const IntegrationRule &ir, - ParMesh *pmesh, - ParFiniteElementSpace &L2, - const ParGridFunction &rho0, - QuadratureData &qdata, - double &volume) -{ - const int NQ = ir.GetNPoints(); - const int Q1D = IntRules.Get(Geometry::SEGMENT,ir.GetOrder()).GetNPoints(); - const int flags = GeometricFactors::JACOBIANS|GeometricFactors::DETERMINANTS; - const GeometricFactors *geom = pmesh->GetGeometricFactors(ir, flags); - Vector rho0Q(NQ*NE); - rho0Q.UseDevice(true); - Vector j, detj; - const QuadratureInterpolator *qi = L2.GetQuadratureInterpolator(ir); - qi->Mult(rho0, QuadratureInterpolator::VALUES, rho0Q, j, detj); - const auto W = ir.GetWeights().Read(); - const auto R = Reshape(rho0Q.Read(), NQ, NE); - const auto J = Reshape(geom->J.Read(), NQ, dim, dim, NE); - const auto detJ = Reshape(geom->detJ.Read(), NQ, NE); - auto V = Reshape(qdata.rho0DetJ0w.Write(), NQ, NE); - Memory &Jinv_m = qdata.Jac0inv.GetMemory(); - const MemoryClass mc = Device::GetMemoryClass(); - const int Ji_total_size = qdata.Jac0inv.TotalSize(); - auto invJ = Reshape(Jinv_m.Write(mc, Ji_total_size), dim, dim, NQ, NE); - Vector vol(NE*NQ), one(NE*NQ); - auto A = Reshape(vol.Write(), NQ, NE); - auto O = Reshape(one.Write(), NQ, NE); - MFEM_ASSERT(dim==2 || dim==3, ""); - if (dim==2) - { - MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + qy * Q1D; - const double J11 = J(q,0,0,e); - const double J12 = J(q,1,0,e); - const double J21 = J(q,0,1,e); - const double J22 = J(q,1,1,e); - const double det = detJ(q,e); - V(q,e) = W[q] * R(q,e) * det; - const double r_idetJ = 1.0 / det; - invJ(0,0,q,e) = J22 * r_idetJ; - invJ(1,0,q,e) = -J12 * r_idetJ; - invJ(0,1,q,e) = -J21 * r_idetJ; - invJ(1,1,q,e) = J11 * r_idetJ; - A(q,e) = W[q] * det; - O(q,e) = 1.0; - } - } - }); - } - else - { - MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, - { - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + (qy + qz * Q1D) * Q1D; - const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e); - const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e); - const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e); - const double det = detJ(q,e); - V(q,e) = W[q] * R(q,e) * det; - const double r_idetJ = 1.0 / det; - invJ(0,0,q,e) = r_idetJ * ((J22 * J33)-(J23 * J32)); - invJ(1,0,q,e) = r_idetJ * ((J32 * J13)-(J33 * J12)); - invJ(2,0,q,e) = r_idetJ * ((J12 * J23)-(J13 * J22)); - invJ(0,1,q,e) = r_idetJ * ((J23 * J31)-(J21 * J33)); - invJ(1,1,q,e) = r_idetJ * ((J33 * J11)-(J31 * J13)); - invJ(2,1,q,e) = r_idetJ * ((J13 * J21)-(J11 * J23)); - invJ(0,2,q,e) = r_idetJ * ((J21 * J32)-(J22 * J31)); - invJ(1,2,q,e) = r_idetJ * ((J31 * J12)-(J32 * J11)); - invJ(2,2,q,e) = r_idetJ * ((J11 * J22)-(J12 * J21)); - A(q,e) = W[q] * det; - O(q,e) = 1.0; - } - } - } - }); - } - qdata.rho0DetJ0w.HostRead(); - volume = vol * one; -} - template static inline void QKernel(const int NE, const int NQ, const bool use_viscosity, diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 4b00600f..9c97b2a7 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -18,13 +18,194 @@ #define MFEM_LAGHOS_SOLVER #include "mfem.hpp" -#include "laghos_assembly.hpp" +#include "laghos_assembly.hpp" // QuadratureData #ifdef MFEM_USE_MPI namespace mfem { +// Choice for the problem setup, statically used in functions below. +static int problem; + +static double gamma_func(const Vector &x) +{ + switch (problem) + { + case 0: return 5.0 / 3.0; + case 1: return 1.4; + case 2: return 1.4; + case 3: return (x(0) > 1.0 && x(1) <= 1.5) ? 1.4 : 1.5; + case 4: return 5.0 / 3.0; + case 5: return 1.4; + case 6: return 1.4; + case 7: return 5.0 / 3.0; + default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; + } +} + +static double rho0(const Vector &x) +{ + switch (problem) + { + case 0: return 1.0; + case 1: return 1.0; + case 2: return (x(0) < 0.5) ? 1.0 : 0.1; + case 3: return (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0; + case 4: return 1.0; + case 5: + { + if (x(0) >= 0.5 && x(1) >= 0.5) { return 0.5313; } + if (x(0) < 0.5 && x(1) < 0.5) { return 0.8; } + return 1.0; + } + case 6: + { + if (x(0) < 0.5 && x(1) >= 0.5) { return 2.0; } + if (x(0) >= 0.5 && x(1) < 0.5) { return 3.0; } + return 1.0; + } + case 7: return x(1) >= 0.0 ? 2.0 : 1.0; + default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; + } +} + +static double radius(double x, double y) { return sqrt(x*x + y*y); } + +static double e0(const Vector &x) +{ + switch (problem) + { + case 0: + { + const double denom = 2.0 / 3.0; // (5/3 - 1) * density. + double val; + if (x.Size() == 2) + { + val = 1.0 + (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) / 4.0; + } + else + { + val = 100.0 + ((cos(2*M_PI*x(2)) + 2) * + (cos(2*M_PI*x(0)) + cos(2*M_PI*x(1))) - 2) / 16.0; + } + return val/denom; + } + case 1: return 0.0; // This case in initialized in main(). + case 2: return (x(0) < 0.5) ? 1.0 / rho0(x) / (gamma_func(x) - 1.0) + : 0.1 / rho0(x) / (gamma_func(x) - 1.0); + case 3: return (x(0) > 1.0) ? 0.1 / rho0(x) / (gamma_func(x) - 1.0) + : 1.0 / rho0(x) / (gamma_func(x) - 1.0); + case 4: + { + const double r = radius(x(0), x(1)), rsq = x(0) * x(0) + x(1) * x(1); + const double gamma = 5.0 / 3.0; + if (r < 0.2) + { + return (5.0 + 25.0 / 2.0 * rsq) / (gamma - 1.0); + } + else if (r < 0.4) + { + const double t1 = 9.0 - 4.0 * log(0.2) + 25.0 / 2.0 * rsq; + const double t2 = 20.0 * r - 4.0 * log(r); + return (t1 - t2) / (gamma - 1.0); + } + else { return (3.0 + 4.0 * log(2.0)) / (gamma - 1.0); } + } + case 5: + { + const double irg = 1.0 / rho0(x) / (gamma_func(x) - 1.0); + if (x(0) >= 0.5 && x(1) >= 0.5) { return 0.4 * irg; } + if (x(0) < 0.5 && x(1) >= 0.5) { return 1.0 * irg; } + if (x(0) < 0.5 && x(1) < 0.5) { return 1.0 * irg; } + if (x(0) >= 0.5 && x(1) < 0.5) { return 1.0 * irg; } + MFEM_ABORT("Error in problem 5!"); + return 0.0; + } + case 6: + { + const double irg = 1.0 / rho0(x) / (gamma_func(x) - 1.0); + if (x(0) >= 0.5 && x(1) >= 0.5) { return 1.0 * irg; } + if (x(0) < 0.5 && x(1) >= 0.5) { return 1.0 * irg; } + if (x(0) < 0.5 && x(1) < 0.5) { return 1.0 * irg; } + if (x(0) >= 0.5 && x(1) < 0.5) { return 1.0 * irg; } + MFEM_ABORT("Error in problem 5!"); + return 0.0; + } + case 7: + { + const double rho = rho0(x), gamma = gamma_func(x); + return (6.0 - rho * x(1)) / (gamma - 1.0) / rho; + } + default: MFEM_ABORT("Bad number given for problem id!"); return 0.0; + } +} + +static void v0(const Vector &x, Vector &v) +{ + const double atn = pow((x(0)*(1.0-x(0))*4*x(1)*(1.0-x(1))*4.0),0.4); + switch (problem) + { + case 0: + v(0) = sin(M_PI*x(0)) * cos(M_PI*x(1)); + v(1) = -cos(M_PI*x(0)) * sin(M_PI*x(1)); + if (x.Size() == 3) + { + v(0) *= cos(M_PI*x(2)); + v(1) *= cos(M_PI*x(2)); + v(2) = 0.0; + } + break; + case 1: v = 0.0; break; + case 2: v = 0.0; break; + case 3: v = 0.0; break; + case 4: + { + v = 0.0; + const double r = radius(x(0), x(1)); + if (r < 0.2) + { + v(0) = 5.0 * x(1); + v(1) = -5.0 * x(0); + } + else if (r < 0.4) + { + v(0) = 2.0 * x(1) / r - 5.0 * x(1); + v(1) = -2.0 * x(0) / r + 5.0 * x(0); + } + else { } + break; + } + case 5: + { + v = 0.0; + if (x(0) >= 0.5 && x(1) >= 0.5) { v(0)=0.0*atn, v(1)=0.0*atn; return;} + if (x(0) < 0.5 && x(1) >= 0.5) { v(0)=0.7276*atn, v(1)=0.0*atn; return;} + if (x(0) < 0.5 && x(1) < 0.5) { v(0)=0.0*atn, v(1)=0.0*atn; return;} + if (x(0) >= 0.5 && x(1) < 0.5) { v(0)=0.0*atn, v(1)=0.7276*atn; return; } + MFEM_ABORT("Error in problem 5!"); + return; + } + case 6: + { + v = 0.0; + if (x(0) >= 0.5 && x(1) >= 0.5) { v(0)=+0.75*atn, v(1)=-0.5*atn; return;} + if (x(0) < 0.5 && x(1) >= 0.5) { v(0)=+0.75*atn, v(1)=+0.5*atn; return;} + if (x(0) < 0.5 && x(1) < 0.5) { v(0)=-0.75*atn, v(1)=+0.5*atn; return;} + if (x(0) >= 0.5 && x(1) < 0.5) { v(0)=-0.75*atn, v(1)=-0.5*atn; return;} + MFEM_ABORT("Error in problem 6!"); + return; + } + case 7: + { + v = 0.0; + v(1) = 0.02 * exp(-2*M_PI*x(1)*x(1)) * cos(2*M_PI*x(0)); + break; + } + default: MFEM_ABORT("Bad number given for problem id!"); + } +} + namespace hydrodynamics { @@ -55,44 +236,7 @@ struct TimingData L2dof(l2d), H1iter(0), L2iter(0), quad_tstep(0) { } }; -class QUpdate -{ -private: - const int dim, vdim, NQ, NE, Q1D; - const bool use_viscosity, use_vorticity; - const double cfl; - TimingData *timer; - const IntegrationRule &ir; - ParFiniteElementSpace &H1, &L2; - const Operator *H1R; - Vector q_dt_est, q_e, e_vec, q_dx, q_dv; - const QuadratureInterpolator *q1,*q2; - const ParGridFunction &gamma_gf; -public: - QUpdate(const int d, const int ne, const int q1d, - const bool visc, const bool vort, - const double cfl, TimingData *t, - const ParGridFunction &gamma_gf, - const IntegrationRule &ir, - ParFiniteElementSpace &h1, ParFiniteElementSpace &l2): - dim(d), vdim(h1.GetVDim()), - NQ(ir.GetNPoints()), NE(ne), Q1D(q1d), - use_viscosity(visc), use_vorticity(vort), cfl(cfl), - timer(t), ir(ir), H1(h1), L2(l2), - H1R(H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC)), - q_dt_est(NE*NQ), - q_e(NE*NQ), - e_vec(NQ*NE*vdim), - q_dx(NQ*NE*vdim*vdim), - q_dv(NQ*NE*vdim*vdim), - q1(H1.GetQuadratureInterpolator(ir)), - q2(L2.GetQuadratureInterpolator(ir)), - gamma_gf(gamma_gf) { } - - void UpdateQuadratureData(const Vector &S, - QuadratureData &qdata, - Vector&,Vector&); -}; +class QUpdate; // Given a solutions state (x, v, e), this class performs all necessary // computations to evaluate the new slopes (dx_dt, dv_dt, de_dt). @@ -225,6 +369,43 @@ class LagrangianHydroOperator : public TimeDependentOperator void AMRUpdate(const Vector&, const bool quick); }; +class QUpdate +{ +private: + const int dim, vdim, NQ, NE, Q1D; + const bool use_viscosity, use_vorticity; + const double cfl; + TimingData *timer; + const IntegrationRule &ir; + ParFiniteElementSpace &H1, &L2; + const Operator *H1R; + Vector q_dt_est, q_e, e_vec, q_dx, q_dv; + const QuadratureInterpolator *q1,*q2; + const ParGridFunction &gamma_gf; +public: + QUpdate(const int d, const int ne, const int q1d, + const bool visc, const bool vort, + const double cfl, TimingData *t, + const ParGridFunction &gamma_gf, + const IntegrationRule &ir, + ParFiniteElementSpace &h1, ParFiniteElementSpace &l2): + dim(d), vdim(h1.GetVDim()), + NQ(ir.GetNPoints()), NE(ne), Q1D(q1d), + use_viscosity(visc), use_vorticity(vort), cfl(cfl), + timer(t), ir(ir), H1(h1), L2(l2), + H1R(H1.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC)), + q_dt_est(NE*NQ), + q_e(NE*NQ), + e_vec(NQ*NE*vdim), + q_dx(NQ*NE*vdim*vdim), + q_dv(NQ*NE*vdim*vdim), + q1(H1.GetQuadratureInterpolator(ir)), + q2(L2.GetQuadratureInterpolator(ir)), + gamma_gf(gamma_gf) { } + + void UpdateQuadratureData(const Vector&, QuadratureData&, Vector&, Vector&); +}; + // TaylorCoefficient used in the 2D Taylor-Green problem. class TaylorCoefficient : public Coefficient { @@ -253,6 +434,140 @@ class RTCoefficient : public VectorCoefficient } // namespace hydrodynamics +namespace amr +{ + +static void Update(BlockVector &S, BlockVector &S_tmp, + Array &true_offset, + ParGridFunction &x_gf, + ParGridFunction &v_gf, + ParGridFunction &e_gf, + ParGridFunction &m_gf) +{ + ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); + ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); + + H1FESpace->Update(); + L2FESpace->Update(); + MEFESpace->Update(); + + const int Vsize_h1 = H1FESpace->GetVSize(); + const int Vsize_l2 = L2FESpace->GetVSize(); + + true_offset[0] = 0; + true_offset[1] = true_offset[0] + Vsize_h1; + true_offset[2] = true_offset[1] + Vsize_h1; + true_offset[3] = true_offset[2] + Vsize_l2; + + S_tmp = S; + S.Update(true_offset); + const Operator* H1Update = H1FESpace->GetUpdateOperator(); + const Operator* L2Update = L2FESpace->GetUpdateOperator(); + H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); + H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); + L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); + + x_gf.MakeRef(H1FESpace, S, true_offset[0]); + v_gf.MakeRef(H1FESpace, S, true_offset[1]); + e_gf.MakeRef(L2FESpace, S, true_offset[2]); + x_gf.SyncAliasMemory(S); + v_gf.SyncAliasMemory(S); + e_gf.SyncAliasMemory(S); + + S_tmp.Update(true_offset); + m_gf.Update(); +} + +} // amr namesapce + +static void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs) +{ + ess_tdofs.SetSize(0); + ess_vdofs.SetSize(0); + Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; + for (int d = 0; d < pmesh->Dimension(); d++) + { + // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, + // i.e., we must enforce v_x/y/z = 0 for the velocity components. + ess_bdr = 0; ess_bdr[d] = 1; + H1.GetEssentialTrueDofs(ess_bdr, dofs_list, d); + ess_tdofs.Append(dofs_list); + H1.GetEssentialVDofs(ess_bdr, dofs_marker, d); + FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); + ess_vdofs.Append(dofs_list); + } +} + +static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements) +{ + Array v; + + for (int i = 0; i < mesh->GetNE(); i++) + { + mesh->GetElementVertices(i, v); + for (int j = 0; j < v.Size(); j++) + { + double dist = 0.0; + for (int l = 0; l < mesh->SpaceDimension(); l++) + { + double d = vert(l) - mesh->GetVertex(v[j])[l]; + dist += d*d; + } + if (dist <= size*size) { elements.Append(i); break; } + } + } +} + +static void Pow(Vector &vec, double p) +{ + for (int i = 0; i < vec.Size(); i++) + { + vec(i) = std::pow(vec(i), p); + } +} + +static void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order = -1) +{ + const FiniteElementSpace *space = gf.FESpace(); + int ne = space->GetNE(); + + if (int_order < 0) { int_order = space->GetOrder(0) + 1; } + + elem_min.SetSize(ne); + elem_max.SetSize(ne); + + Vector vals, tmp; + for (int i = 0; i < ne; i++) + { + int geom = space->GetFE(i)->GetGeomType(); + const IntegrationRule &ir = IntRules.Get(geom, int_order); + + gf.GetValues(i, ir, vals); + + if (space->GetVDim() > 1) + { + Pow(vals, 2.0); + for (int vd = 1; vd < space->GetVDim(); vd++) + { + gf.GetValues(i, ir, tmp, vd+1); + Pow(tmp, 2.0); + vals += tmp; + } + Pow(vals, 0.5); + } + + elem_min(i) = vals.Min(); + elem_max(i) = vals.Max(); + } +} + class HydroODESolver : public ODESolver { protected: From 9420f107f1f35a8ec89fb1b392aa5cd066339933 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 13 Jan 2021 15:41:02 -0800 Subject: [PATCH 12/22] WIP AMR estimators: ZZ, Kelly w/ adhoc integrator --- laghos.cpp | 118 +++++++++++++++++++++++++++++++++++++++----- laghos_assembly.cpp | 87 ++++++++++++++++++++++++++++++++ laghos_assembly.hpp | 23 +++++++++ laghos_solver.hpp | 7 +++ 4 files changed, 223 insertions(+), 12 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 7dcf6b59..b1ab4b62 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -63,8 +63,13 @@ #include #include #include + #include "laghos_solver.hpp" +#undef MFEM_DEBUG_COLOR +#define MFEM_DEBUG_COLOR 199 +#include "general/debug.hpp" + using namespace mfem; static long GetMaxRssMB(); @@ -124,13 +129,13 @@ int main(int argc, char *argv[]) double blast_energy = 0.25; double blast_position[] = {0.0, 0.0, 0.0}; bool amr = false; - int amr_estimator = 0; + int amr_estimator = amr::estimator::std; double amr_ref_threshold = 2e-4; double amr_jac_threshold = 0.92; double amr_deref_threshold = 0.75; int amr_max_level = rs_levels + rp_levels; const double amr_blast_size = 1e-10; - const int amr_nc_limit = 1; + const int amr_nc_limit = 1; // maximum level of hanging nodes OptionsParser args(argc, argv); args.AddOption(&dim, "-dim", "--dimension", "Dimension of the problem."); @@ -292,10 +297,12 @@ int main(int argc, char *argv[]) // Refine the mesh in serial to increase the resolution. if (!amr) { + dbg("Refine the mesh in serial to increase the resolution"); for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } } else { + dbg("RefineAtVertex BLAST"); mesh->EnsureNCMesh(); for (int lev = 0; lev < rs_levels; lev++) { @@ -591,6 +598,55 @@ int main(int argc, char *argv[]) cg_tol, cg_max_iter, ftz_tol, order_q); + // AMR estimator setup + L2_FECollection *flux_fec; + RT_FECollection *smooth_flux_fec; + + ErrorEstimator *estimator = nullptr; + ThresholdRefiner *refiner = nullptr; + ThresholdDerefiner *derefiner = nullptr; + EstimatorIntegrator ei; + + if (amr) + { + const int order = 3; + const int sdim = pmesh->SpaceDimension(); + + dbg("AMR estimator #%d", amr_estimator); + + flux_fec = new L2_FECollection(order, dim); + auto flux_fes = new ParFiniteElementSpace(pmesh, flux_fec, sdim); + + if (amr_estimator == amr::estimator::zz) + { + dbg("ZZ estimator init"); + smooth_flux_fec = new RT_FECollection(order-1, dim); + auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); + estimator = new L2ZienkiewiczZhuEstimator(ei, x_gf, flux_fes, + smooth_flux_fes); + } + + if (amr_estimator == amr::estimator::kelly) + { + dbg("Kelly estimator init"); + estimator = new KellyErrorEstimator(ei, x_gf, flux_fes); + } + + if (estimator) + { + const double max_elem_error = 1.0e-4; + refiner = new ThresholdRefiner(*estimator); + refiner->SetTotalErrorFraction(0.75); + refiner->PreferConformingRefinement(); + refiner->SetNCLimit(amr_nc_limit); + + const double hysteresis = 0.9; // derefinement safety coefficient + derefiner = new ThresholdDerefiner(*estimator); + derefiner->SetThreshold(hysteresis * max_elem_error); + derefiner->SetNCLimit(amr_nc_limit); + } + } + socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; int visport = 19916; @@ -660,6 +716,12 @@ int main(int argc, char *argv[]) t_old = t; hydro.ResetTimeStepEstimate(); + if (amr && refiner && derefiner) + { + refiner->Reset(); + derefiner->Reset(); + } + // S is the vector of dofs, t is the current time, and dt is the time step // to advance. ode_solver->Step(S, t, dt); @@ -788,7 +850,7 @@ int main(int argc, char *argv[]) { Vector v_max, v_min; Array refs; - bool mesh_changed = false; + bool mesh_refined = false; const int NE = pmesh->GetNE(); constexpr double NL_DMAX = std::numeric_limits::max(); @@ -796,7 +858,7 @@ int main(int argc, char *argv[]) switch (amr_estimator) { - case 0: + case amr::estimator::std: { Vector &error_est = hydro.GetZoneMaxVisc(); for (int e = 0; e < NE; e++) @@ -810,7 +872,7 @@ int main(int argc, char *argv[]) } break; } - case 1: + case amr::estimator::rho: { const double art = amr_ref_threshold; const int order = H1FESpace.GetOrder(0) + 1; @@ -848,22 +910,37 @@ int main(int argc, char *argv[]) } break; } - default: MFEM_ABORT("Unknown AMR estimator (should be 0 or 1)!"); + case amr::estimator::zz: + case amr::estimator::kelly: + { + dbg("AMR estimator Apply"); + refiner->Apply(*pmesh); + if (refiner->Refined()) { mesh_refined = true; } + MFEM_VERIFY(!refiner->Derefined(),""); + MFEM_VERIFY(!refiner->Rebalanced(),""); + /*if (refiner->Stop() && myid == 0) + { std::cout << "Stopping criterion satisfied. Stop.\n"; }*/ + break; + } + default: MFEM_ABORT("Unknown AMR estimator!"); } const int nref = pmesh->ReduceInt(refs.Size()); - if (nref) + if (nref && !mesh_refined) { + dbg("GeneralRefinement"); constexpr int non_conforming = 1; pmesh->GeneralRefinement(refs, non_conforming, amr_nc_limit); - mesh_changed = true; + mesh_refined = true; if (myid == 0) { std::cout << "Refined " << nref << " elements." << std::endl; } } - else if (amr_estimator == 0 && amr_deref_threshold >= 0.0) + else if (amr_estimator == amr::estimator::std && + amr_deref_threshold >= 0.0) { + dbg("STD Derefinement"); hydro.ComputeDensity(rho_gf); Vector rho_max, rho_min; @@ -894,16 +971,33 @@ int main(int argc, char *argv[]) } const int op = 2; // maximum value of fine elements - mesh_changed = pmesh->DerefineByError(rho_max, threshold, + mesh_refined = pmesh->DerefineByError(rho_max, threshold, amr_nc_limit, op); - if (mesh_changed && myid == 0) + if (mesh_refined && myid == 0) { std::cout << "Derefined, threshold = " << threshold << std::endl; } } + /*else if ((amr_estimator == amr::estimator::zz || + amr_estimator == amr::estimator::kelly) && + !mesh_refined) + { + dbg("ZZ/KL Derefinement"); + MFEM_VERIFY(derefiner,""); + if (derefiner->Apply(*pmesh)) + { + if (myid == 0) + { + std::cout << "\nDerefined elements." << std::endl; + } + } + if (derefiner->Derefined()) { mesh_refined = true; } + }*/ + else { /* nothing to do */ } - if (mesh_changed) + if (mesh_refined) { + dbg("\033[7mmesh_changed"); constexpr bool quick = true; amr::Update(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index 4edc6124..033f1202 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -23,6 +23,93 @@ using namespace mfem; namespace mfem { +static void ComputeElementFlux0(const FiniteElement &el, + ElementTransformation &Trans, + const Vector &u, + const FiniteElement &fluxelem, + Vector &flux) +{ + const int dof = el.GetDof(); + const int dim = el.GetDim(); + const int sdim = Trans.GetSpaceDim(); + + DenseMatrix dshape(dof, dim); + DenseMatrix invdfdx(dim, sdim); + Vector vec(dim), pointflux(sdim); + + const IntegrationRule &ir = fluxelem.GetNodes(); + const int NQ = ir.GetNPoints(); + flux.SetSize(NQ * sdim); + + for (int q = 0; q < NQ; q++) + { + const IntegrationPoint &ip = ir.IntPoint(q); + el.CalcDShape(ip, dshape); + dshape.MultTranspose(u, vec); + + Trans.SetIntPoint (&ip); + CalcInverse(Trans.Jacobian(), invdfdx); + invdfdx.MultTranspose(vec, pointflux); + + for (int d = 0; d < sdim; d++) + { + flux(NQ*d+q) = pointflux(d); + } + } +} + +static void ComputeElementFlux1(const FiniteElement &el, + ElementTransformation &Trans, + const FiniteElement &fluxelem, + Vector &flux) +{ + const int dim = el.GetDim(); + const int sdim = Trans.GetSpaceDim(); + + DenseMatrix Jadjt(dim, sdim), Jadj(dim, sdim); + + const IntegrationRule &ir = fluxelem.GetNodes(); + const int NQ = ir.GetNPoints(); + flux.SetSize(NQ * sdim); + + constexpr double NL_DMAX = std::numeric_limits::max(); + double minW = +NL_DMAX; + double maxW = -NL_DMAX; + + for (int q = 0; q < NQ; q++) + { + const IntegrationPoint &ip = ir.IntPoint(q); + Trans.SetIntPoint(&ip); + CalcAdjugate(Trans.Jacobian(), Jadj); + Jadjt = Jadj; + Jadjt.Transpose(); + const double w = Jadjt.Weight(); + minW = std::fmin(minW, w); + maxW = std::fmax(maxW, w); + MFEM_VERIFY(std::fabs(maxW) > 1e-13, ""); + const double rho = minW / maxW; + MFEM_VERIFY(rho <= 1.0, ""); + constexpr double amr_jac_threshold = 0.98; + for (int d = 0; d < sdim; d++) + { + const double value = (rho > amr_jac_threshold) ? 0.0: rho; + flux(NQ*d+q) = value; + } + } +} + +void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, + ElementTransformation &Trans, + Vector &u, + const FiniteElement &fluxelem, + Vector &flux, + bool with_coef) +{ + // ZZ comes with with_coef set to true, not Kelly + // ComputeElementFlux0(el, Trans, u, fluxelem, flux); + ComputeElementFlux1(el, Trans, fluxelem, flux); +} + namespace hydrodynamics { diff --git a/laghos_assembly.hpp b/laghos_assembly.hpp index ba577d52..7dde15f1 100644 --- a/laghos_assembly.hpp +++ b/laghos_assembly.hpp @@ -17,11 +17,34 @@ #ifndef MFEM_LAGHOS_ASSEMBLY #define MFEM_LAGHOS_ASSEMBLY +#define MFEM_DEBUG_COLOR 226 +#include "general/debug.hpp" + #include "mfem.hpp" namespace mfem { +class EstimatorIntegrator: public BilinearFormIntegrator +{ +public: + EstimatorIntegrator() { dbg(); } + + void AssemblePA(const FiniteElementSpace &fes) {dbg();} + + double ComputeFluxEnergy(const FiniteElement &fluxelem, + ElementTransformation &Trans, + Vector &flux, Vector *d_energy = NULL) + { dbg(); return 0.0; } + + virtual void ComputeElementFlux(const FiniteElement &el, + ElementTransformation &Trans, + Vector &u, + const FiniteElement &fluxelem, + Vector &flux, + bool with_coef = true); +}; + namespace hydrodynamics { diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 9c97b2a7..61c6953c 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -304,6 +304,7 @@ class LagrangianHydroOperator : public TimeDependentOperator mutable Array c_tdofs[3]; mutable Vector zone_max_visc, zone_vgrad; + virtual void ComputeMaterialProperties(int nvalues, const double gamma[], const double rho[], const double e[], double p[], double cs[]) const @@ -437,6 +438,8 @@ class RTCoefficient : public VectorCoefficient namespace amr { +enum estimator: int { std = 0, rho = 1, zz = 2, kelly = 3 }; + static void Update(BlockVector &S, BlockVector &S_tmp, Array &true_offset, ParGridFunction &x_gf, @@ -477,6 +480,10 @@ static void Update(BlockVector &S, BlockVector &S_tmp, S_tmp.Update(true_offset); m_gf.Update(); + + //H1FESpace->UpdatesFinished(); + //L2FESpace->UpdatesFinished(); + //MEFESpace->UpdatesFinished(); } } // amr namesapce From d88404a2426248ed167235717948a66a88453b3e Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 19 Jan 2021 14:00:46 -0800 Subject: [PATCH 13/22] Cleanup and added laghos_amr files --- laghos.cpp | 235 ++----------------------- laghos_amr.cpp | 416 ++++++++++++++++++++++++++++++++++++++++++++ laghos_amr.hpp | 233 +++++++++++++++++++++++++ laghos_assembly.cpp | 87 --------- laghos_assembly.hpp | 23 --- laghos_solver.cpp | 1 - laghos_solver.hpp | 162 +++-------------- 7 files changed, 688 insertions(+), 469 deletions(-) create mode 100644 laghos_amr.cpp create mode 100644 laghos_amr.hpp diff --git a/laghos.cpp b/laghos.cpp index b1ab4b62..6f3b3686 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -64,6 +64,7 @@ #include #include +#include "laghos_amr.hpp" #include "laghos_solver.hpp" #undef MFEM_DEBUG_COLOR @@ -598,54 +599,17 @@ int main(int argc, char *argv[]) cg_tol, cg_max_iter, ftz_tol, order_q); - // AMR estimator setup - L2_FECollection *flux_fec; - RT_FECollection *smooth_flux_fec; - - ErrorEstimator *estimator = nullptr; - ThresholdRefiner *refiner = nullptr; - ThresholdDerefiner *derefiner = nullptr; - EstimatorIntegrator ei; - - if (amr) - { - const int order = 3; - const int sdim = pmesh->SpaceDimension(); - - dbg("AMR estimator #%d", amr_estimator); - - flux_fec = new L2_FECollection(order, dim); - auto flux_fes = new ParFiniteElementSpace(pmesh, flux_fec, sdim); - - if (amr_estimator == amr::estimator::zz) - { - dbg("ZZ estimator init"); - smooth_flux_fec = new RT_FECollection(order-1, dim); - auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); - estimator = new L2ZienkiewiczZhuEstimator(ei, x_gf, flux_fes, - smooth_flux_fes); - } - - if (amr_estimator == amr::estimator::kelly) - { - dbg("Kelly estimator init"); - estimator = new KellyErrorEstimator(ei, x_gf, flux_fes); - } - - if (estimator) - { - const double max_elem_error = 1.0e-4; - refiner = new ThresholdRefiner(*estimator); - refiner->SetTotalErrorFraction(0.75); - refiner->PreferConformingRefinement(); - refiner->SetNCLimit(amr_nc_limit); - - const double hysteresis = 0.9; // derefinement safety coefficient - derefiner = new ThresholdDerefiner(*estimator); - derefiner->SetThreshold(hysteresis * max_elem_error); - derefiner->SetNCLimit(amr_nc_limit); - } - } + amr::Operator *AMR = amr ? new amr::Operator(pmesh, + amr_estimator, + amr_ref_threshold, + amr_jac_threshold, + amr_deref_threshold, + amr_max_level, + amr_blast_size, + amr_nc_limit, + blast_energy, + blast_position) : nullptr; + if (amr) { AMR->Setup(x_gf); } socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; @@ -716,11 +680,7 @@ int main(int argc, char *argv[]) t_old = t; hydro.ResetTimeStepEstimate(); - if (amr && refiner && derefiner) - { - refiner->Reset(); - derefiner->Reset(); - } + if (amr) { AMR->Reset(); } // S is the vector of dofs, t is the current time, and dt is the time step // to advance. @@ -848,170 +808,11 @@ int main(int argc, char *argv[]) if (amr) { - Vector v_max, v_min; - Array refs; - bool mesh_refined = false; - const int NE = pmesh->GetNE(); - constexpr double NL_DMAX = std::numeric_limits::max(); - - GetPerElementMinMax(v_gf, v_min, v_max); - - switch (amr_estimator) - { - case amr::estimator::std: - { - Vector &error_est = hydro.GetZoneMaxVisc(); - for (int e = 0; e < NE; e++) - { - if (error_est(e) > amr_ref_threshold && - pmesh->pncmesh->GetElementDepth(e) < amr_max_level && - (v_min(e) < 1e-3)) // only refine the still area - { - refs.Append(Refinement(e)); - } - } - break; - } - case amr::estimator::rho: - { - const double art = amr_ref_threshold; - const int order = H1FESpace.GetOrder(0) + 1; - DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); - MFEM_VERIFY(art >= 0.0, "AMR threshold should be positive"); - for (int e = 0; e < NE; e++) - { - double minW = +NL_DMAX; - double maxW = -NL_DMAX; - const int depth = pmesh->pncmesh->GetElementDepth(e); - ElementTransformation *eTr = pmesh->GetElementTransformation(e); - const Geometry::Type &type = pmesh->GetElement(e)->GetGeometryType(); - const IntegrationRule *ir = &IntRules.Get(type, order); - const int NQ = ir->GetNPoints(); - for (int q = 0; q < NQ; q++) - { - eTr->SetIntPoint(&ir->IntPoint(q)); - const DenseMatrix &J = eTr->Jacobian(); - CalcAdjugate(J, Jadj); - Jadjt = Jadj; - Jadjt.Transpose(); - const double w = Jadjt.Weight(); - minW = std::fmin(minW, w); - maxW = std::fmax(maxW, w); - } - if (std::fabs(maxW) != 0.0) - { - const double rho = minW / maxW; - MFEM_VERIFY(rho <= 1.0, ""); - if (rho < amr_jac_threshold && depth < amr_max_level) - { - refs.Append(Refinement(e)); - } - } - } - break; - } - case amr::estimator::zz: - case amr::estimator::kelly: - { - dbg("AMR estimator Apply"); - refiner->Apply(*pmesh); - if (refiner->Refined()) { mesh_refined = true; } - MFEM_VERIFY(!refiner->Derefined(),""); - MFEM_VERIFY(!refiner->Rebalanced(),""); - /*if (refiner->Stop() && myid == 0) - { std::cout << "Stopping criterion satisfied. Stop.\n"; }*/ - break; - } - default: MFEM_ABORT("Unknown AMR estimator!"); - } - - const int nref = pmesh->ReduceInt(refs.Size()); - if (nref && !mesh_refined) - { - dbg("GeneralRefinement"); - constexpr int non_conforming = 1; - pmesh->GeneralRefinement(refs, non_conforming, amr_nc_limit); - mesh_refined = true; - if (myid == 0) - { - std::cout << "Refined " << nref << " elements." << std::endl; - } - } - else if (amr_estimator == amr::estimator::std && - amr_deref_threshold >= 0.0) - { - dbg("STD Derefinement"); - hydro.ComputeDensity(rho_gf); - - Vector rho_max, rho_min; - GetPerElementMinMax(rho_gf, rho_min, rho_max); - - // Derefinement based on zone maximum rho in post-shock region - const double rho_max_max = rho_max.Size() ? rho_max.Max() : 0.0; - double threshold, loc_threshold = amr_deref_threshold * rho_max_max; - MPI_Allreduce(&loc_threshold, &threshold, 1, MPI_DOUBLE, MPI_MAX, - pmesh->GetComm()); - - // make sure the blast point is never derefined - Array elements; - FindElementsWithVertex(pmesh, Vertex(blast_position[0], - blast_position[1], - blast_position[2]), - amr_blast_size, elements); - for (int i = 0; i < elements.Size(); i++) - { - int index = elements[i]; - if (index >= 0) { rho_max(index) = NL_DMAX; } - } - - // only derefine where the mesh is in motion, i.e. after the shock - for (int i = 0; i < pmesh->GetNE(); i++) - { - if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } - } - - const int op = 2; // maximum value of fine elements - mesh_refined = pmesh->DerefineByError(rho_max, threshold, - amr_nc_limit, op); - if (mesh_refined && myid == 0) - { - std::cout << "Derefined, threshold = " << threshold << std::endl; - } - } - /*else if ((amr_estimator == amr::estimator::zz || - amr_estimator == amr::estimator::kelly) && - !mesh_refined) - { - dbg("ZZ/KL Derefinement"); - MFEM_VERIFY(derefiner,""); - if (derefiner->Apply(*pmesh)) - { - if (myid == 0) - { - std::cout << "\nDerefined elements." << std::endl; - } - } - if (derefiner->Derefined()) { mesh_refined = true; } - }*/ - else { /* nothing to do */ } - - if (mesh_refined) - { - dbg("\033[7mmesh_changed"); - constexpr bool quick = true; - - amr::Update(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); - hydro.AMRUpdate(S, quick); - - pmesh->Rebalance(); - - amr::Update(S, S_old, true_offset, x_gf, v_gf, e_gf, m_gf); - hydro.AMRUpdate(S, !quick); - - GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); - ode_solver->Init(hydro); - //H1FESpace.PrintPartitionStats(); - } + AMR->Update(S, S_old, + x_gf,v_gf,e_gf,m_gf, + true_offset, + hydro, ode_solver, + bdr_attr_max, ess_tdofs, ess_vdofs); } // Problems checks diff --git a/laghos_amr.cpp b/laghos_amr.cpp new file mode 100644 index 00000000..95753744 --- /dev/null +++ b/laghos_amr.cpp @@ -0,0 +1,416 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +#define MFEM_DEBUG_COLOR 226 +#include "general/debug.hpp" + +#include "laghos_solver.hpp" +#include "laghos_amr.hpp" + +namespace mfem +{ + +namespace amr +{ + +static void ComputeElementFlux0(const FiniteElement &el, + ElementTransformation &Trans, + const Vector &u, + const FiniteElement &fluxelem, + Vector &flux) +{ + const int dof = el.GetDof(); + const int dim = el.GetDim(); + const int sdim = Trans.GetSpaceDim(); + + DenseMatrix dshape(dof, dim); + DenseMatrix invdfdx(dim, sdim); + Vector vec(dim), pointflux(sdim); + + const IntegrationRule &ir = fluxelem.GetNodes(); + const int NQ = ir.GetNPoints(); + flux.SetSize(NQ * sdim); + + for (int q = 0; q < NQ; q++) + { + const IntegrationPoint &ip = ir.IntPoint(q); + el.CalcDShape(ip, dshape); + dshape.MultTranspose(u, vec); + + Trans.SetIntPoint (&ip); + CalcInverse(Trans.Jacobian(), invdfdx); + invdfdx.MultTranspose(vec, pointflux); + + for (int d = 0; d < sdim; d++) + { + flux(NQ*d+q) = pointflux(d); + } + } +} + +static void ComputeElementFlux1(const FiniteElement &el, + ElementTransformation &Trans, + const FiniteElement &fluxelem, + Vector &flux) +{ + const int dim = el.GetDim(); + const int sdim = Trans.GetSpaceDim(); + + DenseMatrix Jadjt(dim, sdim), Jadj(dim, sdim); + + const IntegrationRule &ir = fluxelem.GetNodes(); + const int NQ = ir.GetNPoints(); + flux.SetSize(NQ * sdim); + + constexpr double NL_DMAX = std::numeric_limits::max(); + double minW = +NL_DMAX; + double maxW = -NL_DMAX; + + for (int q = 0; q < NQ; q++) + { + const IntegrationPoint &ip = ir.IntPoint(q); + Trans.SetIntPoint(&ip); + CalcAdjugate(Trans.Jacobian(), Jadj); + Jadjt = Jadj; + Jadjt.Transpose(); + const double w = Jadjt.Weight(); + minW = std::fmin(minW, w); + maxW = std::fmax(maxW, w); + MFEM_VERIFY(std::fabs(maxW) > 1e-13, ""); + const double rho = minW / maxW; + MFEM_VERIFY(rho <= 1.0, ""); + constexpr double amr_jac_threshold = 0.98; + for (int d = 0; d < sdim; d++) + { + const double value = (rho > amr_jac_threshold) ? 0.0: rho; + flux(NQ*d+q) = value; + } + } +} + +void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, + ElementTransformation &Trans, + Vector &u, + const FiniteElement &fluxelem, + Vector &flux, + bool with_coef) +{ + // ZZ comes with with_coef set to true, not Kelly + switch (flux_mode) + { + case mode::diffusion: + { + DiffusionIntegrator::ComputeElementFlux(el, Trans, u, + fluxelem, flux, with_coef); + break; + } + case mode::one: + { + ComputeElementFlux0(el, Trans, u, fluxelem, flux); + break; + } + case mode::two: + { + ComputeElementFlux1(el, Trans, fluxelem, flux); + break; + } + default: MFEM_ABORT("Unknown mode!"); + } +} + +void Eval(const ParGridFunction &v_gf, const double threshold, + ParMesh &pmesh, Array &refs) +{ + Vector val; + IntegrationPoint ip; + const int NE = pmesh.GetNE(); + const int dim = pmesh.Dimension(); + const int kmax = dim - 2; + constexpr double w = 1.0; + + //ParFiniteElementSpace *pfes = v_gf.ParFESpace(); + + for (int e = 0; e < NE; e++) + { + for (int x1 = 0; x1 <= 1; x1++) + { + for (int x2 = 0; x2 <= 1; x2++) + { + for (int x3 = 0; x3 <= kmax; x3++) + { + ip.Set(x1, x2, x3, w); + v_gf.GetVectorValue(e, ip, val); + const double l2_norm = val.Norml2(); + if (l2_norm > threshold) { refs.Append(Refinement(e)); } + } + } + } + } +} + +Operator::Operator(ParMesh *pmesh, + int estimator, + double ref_t, + double jac_t, + double deref_t, + int max_level, + const double blast_size, + const int nc_limit, + const double blast_energy, + const double *blast_xyz): + pmesh(pmesh), + myid(pmesh->GetMyRank()), + dim(pmesh->Dimension()), + sdim(pmesh->SpaceDimension()), + opt( +{ + estimator, ref_t, jac_t, deref_t, max_level, blast_size, nc_limit, + blast_energy, Vertex(blast_xyz[0], blast_xyz[1], blast_xyz[2]) +}) +{ + dbg(); +} + +Operator::~Operator() { dbg(); } + +void Operator::Setup(ParGridFunction &u) +{ + dbg("AMR estimator #%d", opt.estimator); + + flux_fec = new L2_FECollection(order, dim); + auto flux_fes = new ParFiniteElementSpace(pmesh, flux_fec, sdim); + + if (opt.estimator == amr::estimator::zz) + { + dbg("ZZ estimator init"); + ei = new amr::EstimatorIntegrator(); + smooth_flux_fec = new RT_FECollection(order-1, dim); + auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); + estimator = new L2ZienkiewiczZhuEstimator(*ei, u, flux_fes, + smooth_flux_fes); + } + + if (opt.estimator == amr::estimator::kelly) + { + dbg("Kelly estimator init"); + ei = new amr::EstimatorIntegrator(); + estimator = new KellyErrorEstimator(*ei, u, flux_fes); + } + + if (estimator) + { + const double max_elem_error = 1.0e-4; + refiner = new ThresholdRefiner(*estimator); + refiner->SetTotalErrorFraction(0.75); + refiner->PreferConformingRefinement(); + refiner->SetNCLimit(opt.nc_limit); + + const double hysteresis = 0.9; // derefinement safety coefficient + derefiner = new ThresholdDerefiner(*estimator); + derefiner->SetThreshold(hysteresis * max_elem_error); + derefiner->SetNCLimit(opt.nc_limit); + } +} + +void Operator::Reset() +{ + if (!(refiner && derefiner)) { return; } + refiner->Reset(); + derefiner->Reset(); +} + +void Operator::Update(BlockVector &S, + BlockVector &S_old, + ParGridFunction &x, + ParGridFunction &v, + ParGridFunction &e, + ParGridFunction &m, + Array &true_offset, + hydrodynamics::LagrangianHydroOperator &hydro, + ODESolver *ode_solver, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs) +{ + Vector v_max, v_min; + Array refs; + bool mesh_refined = false; + const int NE = pmesh->GetNE(); + ParFiniteElementSpace &H1FESpace = *x.ParFESpace(); + constexpr double NL_DMAX = std::numeric_limits::max(); + + GetPerElementMinMax(v, v_min, v_max); + + switch (opt.estimator) + { + case amr::estimator::std: + { + Vector &error_est = hydro.GetZoneMaxVisc(); + for (int e = 0; e < NE; e++) + { + if (error_est(e) > opt.ref_threshold && + pmesh->pncmesh->GetElementDepth(e) < opt.max_level && + (v_min(e) < 1e-3)) // only refine the still area + { + refs.Append(Refinement(e)); + } + } + break; + } + case amr::estimator::rho: + { + const double art = opt.ref_threshold; + const int order = H1FESpace.GetOrder(0) + 1; + DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); + MFEM_VERIFY(art >= 0.0, "AMR threshold should be positive"); + for (int e = 0; e < NE; e++) + { + double minW = +NL_DMAX; + double maxW = -NL_DMAX; + const int depth = pmesh->pncmesh->GetElementDepth(e); + ElementTransformation *eTr = pmesh->GetElementTransformation(e); + const Geometry::Type &type = pmesh->GetElement(e)->GetGeometryType(); + const IntegrationRule *ir = &IntRules.Get(type, order); + const int NQ = ir->GetNPoints(); + for (int q = 0; q < NQ; q++) + { + eTr->SetIntPoint(&ir->IntPoint(q)); + const DenseMatrix &J = eTr->Jacobian(); + CalcAdjugate(J, Jadj); + Jadjt = Jadj; + Jadjt.Transpose(); + const double w = Jadjt.Weight(); + minW = std::fmin(minW, w); + maxW = std::fmax(maxW, w); + } + if (std::fabs(maxW) != 0.0) + { + const double rho = minW / maxW; + MFEM_VERIFY(rho <= 1.0, ""); + if (rho < opt.jac_threshold && depth < opt.max_level) + { + refs.Append(Refinement(e)); + } + } + } + break; + } + case amr::estimator::zz: + case amr::estimator::kelly: + { + dbg("AMR estimator Apply"); + refiner->Apply(*pmesh); + if (refiner->Refined()) { mesh_refined = true; } + MFEM_VERIFY(!refiner->Derefined(),""); + MFEM_VERIFY(!refiner->Rebalanced(),""); + /*if (refiner->Stop() && myid == 0) + { std::cout << "Stopping criterion satisfied. Stop.\n"; }*/ + break; + } + default: MFEM_ABORT("Unknown AMR estimator!"); + } + + const int nref = pmesh->ReduceInt(refs.Size()); + if (nref && !mesh_refined) + { + dbg("GeneralRefinement"); + constexpr int non_conforming = 1; + pmesh->GeneralRefinement(refs, non_conforming, opt.nc_limit); + mesh_refined = true; + if (myid == 0) + { + std::cout << "Refined " << nref << " elements." << std::endl; + } + } + else if (opt.estimator == amr::estimator::std && + opt.deref_threshold >= 0.0) + { + dbg("STD Derefinement"); + ParGridFunction rho_gf; + hydro.ComputeDensity(rho_gf); + + Vector rho_max, rho_min; + GetPerElementMinMax(rho_gf, rho_min, rho_max); + + // Derefinement based on zone maximum rho in post-shock region + const double rho_max_max = rho_max.Size() ? rho_max.Max() : 0.0; + double threshold, loc_threshold = opt.deref_threshold * rho_max_max; + MPI_Allreduce(&loc_threshold, &threshold, 1, MPI_DOUBLE, MPI_MAX, + pmesh->GetComm()); + + // make sure the blast point is never derefined + Array elements; + FindElementsWithVertex(pmesh, opt.blast_position, + opt.blast_size, elements); + for (int i = 0; i < elements.Size(); i++) + { + int index = elements[i]; + if (index >= 0) { rho_max(index) = NL_DMAX; } + } + + // only derefine where the mesh is in motion, i.e. after the shock + for (int i = 0; i < pmesh->GetNE(); i++) + { + if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } + } + + const int op = 2; // maximum value of fine elements + mesh_refined = pmesh->DerefineByError(rho_max, threshold, + opt.nc_limit, op); + if (mesh_refined && myid == 0) + { + std::cout << "Derefined, threshold = " << threshold << std::endl; + } + } + /*else if ((amr_estimator == amr::estimator::zz || + amr_estimator == amr::estimator::kelly) && + !mesh_refined) + { + dbg("ZZ/KL Derefinement"); + MFEM_VERIFY(derefiner,""); + if (derefiner->Apply(*pmesh)) + { + if (myid == 0) + { + std::cout << "\nDerefined elements." << std::endl; + } + } + if (derefiner->Derefined()) { mesh_refined = true; } + }*/ + else { /* nothing to do */ } + + if (mesh_refined) + { + dbg("\033[7mmesh_changed"); + constexpr bool quick = true; + + amr::Update(S, S_old, true_offset, x, v, e, m); + hydro.AMRUpdate(S, quick); + + pmesh->Rebalance(); + + amr::Update(S, S_old, true_offset, x, v, e, m); + hydro.AMRUpdate(S, !quick); + + GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); + ode_solver->Init(hydro); + //H1FESpace.PrintPartitionStats(); + } +} + +} // namespace amr + +} // namespace mfem diff --git a/laghos_amr.hpp b/laghos_amr.hpp new file mode 100644 index 00000000..5f62e370 --- /dev/null +++ b/laghos_amr.hpp @@ -0,0 +1,233 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at +// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights +// reserved. See files LICENSE and NOTICE for details. +// +// This file is part of CEED, a collection of benchmarks, miniapps, software +// libraries and APIs for efficient high-order finite element and spectral +// element discretizations for exascale applications. For more information and +// source code availability see http://github.com/ceed. +// +// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, +// a collaborative effort of two U.S. Department of Energy organizations (Office +// of Science and the National Nuclear Security Administration) responsible for +// the planning and preparation of a capable exascale ecosystem, including +// software, applications, hardware, advanced system engineering and early +// testbed platforms, in support of the nation's exascale computing imperative. + +#ifndef MFEM_LAGHOS_AMR +#define MFEM_LAGHOS_AMR + +#include "mfem.hpp" +#include "laghos_solver.hpp" + +namespace mfem +{ + +namespace amr +{ + +enum estimator: int { std = 0, rho = 1, zz = 2, kelly = 3 }; + + +static void Update(BlockVector &S, BlockVector &S_tmp, + Array &true_offset, + ParGridFunction &x_gf, + ParGridFunction &v_gf, + ParGridFunction &e_gf, + ParGridFunction &m_gf) +{ + ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); + ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); + + H1FESpace->Update(); + L2FESpace->Update(); + MEFESpace->Update(); + + const int Vsize_h1 = H1FESpace->GetVSize(); + const int Vsize_l2 = L2FESpace->GetVSize(); + + true_offset[0] = 0; + true_offset[1] = true_offset[0] + Vsize_h1; + true_offset[2] = true_offset[1] + Vsize_h1; + true_offset[3] = true_offset[2] + Vsize_l2; + + S_tmp = S; + S.Update(true_offset); + const Operator* H1Update = H1FESpace->GetUpdateOperator(); + const Operator* L2Update = L2FESpace->GetUpdateOperator(); + H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); + H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); + L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); + + x_gf.MakeRef(H1FESpace, S, true_offset[0]); + v_gf.MakeRef(H1FESpace, S, true_offset[1]); + e_gf.MakeRef(L2FESpace, S, true_offset[2]); + x_gf.SyncAliasMemory(S); + v_gf.SyncAliasMemory(S); + e_gf.SyncAliasMemory(S); + + S_tmp.Update(true_offset); + m_gf.Update(); + + //H1FESpace->UpdatesFinished(); + //L2FESpace->UpdatesFinished(); + //MEFESpace->UpdatesFinished(); +} + +static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements) +{ + Array v; + + for (int i = 0; i < mesh->GetNE(); i++) + { + mesh->GetElementVertices(i, v); + for (int j = 0; j < v.Size(); j++) + { + double dist = 0.0; + for (int l = 0; l < mesh->SpaceDimension(); l++) + { + double d = vert(l) - mesh->GetVertex(v[j])[l]; + dist += d*d; + } + if (dist <= size*size) { elements.Append(i); break; } + } + } +} + +static void Pow(Vector &vec, double p) +{ + for (int i = 0; i < vec.Size(); i++) + { + vec(i) = std::pow(vec(i), p); + } +} + +static void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order = -1) +{ + const FiniteElementSpace *space = gf.FESpace(); + int ne = space->GetNE(); + + if (int_order < 0) { int_order = space->GetOrder(0) + 1; } + + elem_min.SetSize(ne); + elem_max.SetSize(ne); + + Vector vals, tmp; + for (int i = 0; i < ne; i++) + { + int geom = space->GetFE(i)->GetGeomType(); + const IntegrationRule &ir = IntRules.Get(geom, int_order); + + gf.GetValues(i, ir, vals); + + if (space->GetVDim() > 1) + { + Pow(vals, 2.0); + for (int vd = 1; vd < space->GetVDim(); vd++) + { + gf.GetValues(i, ir, tmp, vd+1); + Pow(tmp, 2.0); + vals += tmp; + } + Pow(vals, 0.5); + } + + elem_min(i) = vals.Min(); + elem_max(i) = vals.Max(); + } +} + +class EstimatorIntegrator: public DiffusionIntegrator +{ + enum class mode { diffusion, one, two }; + const mode flux_mode; + ConstantCoefficient one {1.0}; +public: + EstimatorIntegrator(const mode flux_mode = mode::diffusion): + DiffusionIntegrator(one), flux_mode(flux_mode) { } + + double ComputeFluxEnergy(const FiniteElement &fluxelem, + ElementTransformation &Trans, + Vector &flux, Vector *d_energy = NULL) + { + if (flux_mode == mode::diffusion) + { + return DiffusionIntegrator::ComputeFluxEnergy(fluxelem, Trans, flux, d_energy); + } + // Not implemented for other modes + MFEM_ABORT("Not implemented!"); + return 0.0; + } + + virtual void ComputeElementFlux(const FiniteElement &el, + ElementTransformation &Trans, + Vector &u, + const FiniteElement &fluxelem, + Vector &flux, + bool with_coef = false); +}; + +class Operator +{ + const int order = 3; + ParMesh *pmesh; + const int myid, dim, sdim; + const struct Options + { + int estimator; + double ref_threshold; + double jac_threshold; + double deref_threshold; + int max_level; + double blast_size; + int nc_limit; + double blast_energy; + Vertex blast_position; + } opt; + + // AMR estimator setup + L2_FECollection *flux_fec; + RT_FECollection *smooth_flux_fec; + + ErrorEstimator *estimator = nullptr; + ThresholdRefiner *refiner = nullptr; + ThresholdDerefiner *derefiner = nullptr; + amr::EstimatorIntegrator *ei = nullptr; + +public: + Operator(ParMesh *pmesh, + int estimator, + double ref_t, double jac_t, double deref_t, + int max_level, + const double blast_size, const int nc_limit, + const double blast_energy, const double *blast_position); + + ~Operator(); + + void Setup(ParGridFunction&); + + void Reset(); + + void Update(BlockVector &S, + BlockVector &S_old, + ParGridFunction &x, + ParGridFunction &v, + ParGridFunction &e, + ParGridFunction &m, + Array &true_offset, + hydrodynamics::LagrangianHydroOperator&, + ODESolver*, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs); +}; + +} // namespace amr + +} // namespace mfem + +#endif // MFEM_LAGHOS_AMR diff --git a/laghos_assembly.cpp b/laghos_assembly.cpp index 033f1202..4edc6124 100644 --- a/laghos_assembly.cpp +++ b/laghos_assembly.cpp @@ -23,93 +23,6 @@ using namespace mfem; namespace mfem { -static void ComputeElementFlux0(const FiniteElement &el, - ElementTransformation &Trans, - const Vector &u, - const FiniteElement &fluxelem, - Vector &flux) -{ - const int dof = el.GetDof(); - const int dim = el.GetDim(); - const int sdim = Trans.GetSpaceDim(); - - DenseMatrix dshape(dof, dim); - DenseMatrix invdfdx(dim, sdim); - Vector vec(dim), pointflux(sdim); - - const IntegrationRule &ir = fluxelem.GetNodes(); - const int NQ = ir.GetNPoints(); - flux.SetSize(NQ * sdim); - - for (int q = 0; q < NQ; q++) - { - const IntegrationPoint &ip = ir.IntPoint(q); - el.CalcDShape(ip, dshape); - dshape.MultTranspose(u, vec); - - Trans.SetIntPoint (&ip); - CalcInverse(Trans.Jacobian(), invdfdx); - invdfdx.MultTranspose(vec, pointflux); - - for (int d = 0; d < sdim; d++) - { - flux(NQ*d+q) = pointflux(d); - } - } -} - -static void ComputeElementFlux1(const FiniteElement &el, - ElementTransformation &Trans, - const FiniteElement &fluxelem, - Vector &flux) -{ - const int dim = el.GetDim(); - const int sdim = Trans.GetSpaceDim(); - - DenseMatrix Jadjt(dim, sdim), Jadj(dim, sdim); - - const IntegrationRule &ir = fluxelem.GetNodes(); - const int NQ = ir.GetNPoints(); - flux.SetSize(NQ * sdim); - - constexpr double NL_DMAX = std::numeric_limits::max(); - double minW = +NL_DMAX; - double maxW = -NL_DMAX; - - for (int q = 0; q < NQ; q++) - { - const IntegrationPoint &ip = ir.IntPoint(q); - Trans.SetIntPoint(&ip); - CalcAdjugate(Trans.Jacobian(), Jadj); - Jadjt = Jadj; - Jadjt.Transpose(); - const double w = Jadjt.Weight(); - minW = std::fmin(minW, w); - maxW = std::fmax(maxW, w); - MFEM_VERIFY(std::fabs(maxW) > 1e-13, ""); - const double rho = minW / maxW; - MFEM_VERIFY(rho <= 1.0, ""); - constexpr double amr_jac_threshold = 0.98; - for (int d = 0; d < sdim; d++) - { - const double value = (rho > amr_jac_threshold) ? 0.0: rho; - flux(NQ*d+q) = value; - } - } -} - -void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, - ElementTransformation &Trans, - Vector &u, - const FiniteElement &fluxelem, - Vector &flux, - bool with_coef) -{ - // ZZ comes with with_coef set to true, not Kelly - // ComputeElementFlux0(el, Trans, u, fluxelem, flux); - ComputeElementFlux1(el, Trans, fluxelem, flux); -} - namespace hydrodynamics { diff --git a/laghos_assembly.hpp b/laghos_assembly.hpp index 7dde15f1..ba577d52 100644 --- a/laghos_assembly.hpp +++ b/laghos_assembly.hpp @@ -17,34 +17,11 @@ #ifndef MFEM_LAGHOS_ASSEMBLY #define MFEM_LAGHOS_ASSEMBLY -#define MFEM_DEBUG_COLOR 226 -#include "general/debug.hpp" - #include "mfem.hpp" namespace mfem { -class EstimatorIntegrator: public BilinearFormIntegrator -{ -public: - EstimatorIntegrator() { dbg(); } - - void AssemblePA(const FiniteElementSpace &fes) {dbg();} - - double ComputeFluxEnergy(const FiniteElement &fluxelem, - ElementTransformation &Trans, - Vector &flux, Vector *d_energy = NULL) - { dbg(); return 0.0; } - - virtual void ComputeElementFlux(const FiniteElement &el, - ElementTransformation &Trans, - Vector &u, - const FiniteElement &fluxelem, - Vector &flux, - bool with_coef = true); -}; - namespace hydrodynamics { diff --git a/laghos_solver.cpp b/laghos_solver.cpp index a81b9682..5df43d5c 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -681,7 +681,6 @@ void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) const } } - double LagrangianHydroOperator::InternalEnergy(const ParGridFunction &gf) const { double glob_ie = 0.0; diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 61c6953c..4b372a5e 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -206,6 +206,27 @@ static void v0(const Vector &x, Vector &v) } } +static void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs) +{ + ess_tdofs.SetSize(0); + ess_vdofs.SetSize(0); + Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; + for (int d = 0; d < pmesh->Dimension(); d++) + { + // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, + // i.e., we must enforce v_x/y/z = 0 for the velocity components. + ess_bdr = 0; ess_bdr[d] = 1; + H1.GetEssentialTrueDofs(ess_bdr, dofs_list, d); + ess_tdofs.Append(dofs_list); + H1.GetEssentialVDofs(ess_bdr, dofs_marker, d); + FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); + ess_vdofs.Append(dofs_list); + } +} + namespace hydrodynamics { @@ -304,7 +325,6 @@ class LagrangianHydroOperator : public TimeDependentOperator mutable Array c_tdofs[3]; mutable Vector zone_max_visc, zone_vgrad; - virtual void ComputeMaterialProperties(int nvalues, const double gamma[], const double rho[], const double e[], double p[], double cs[]) const @@ -435,146 +455,6 @@ class RTCoefficient : public VectorCoefficient } // namespace hydrodynamics -namespace amr -{ - -enum estimator: int { std = 0, rho = 1, zz = 2, kelly = 3 }; - -static void Update(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf, - ParGridFunction &m_gf) -{ - ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); - ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); - - H1FESpace->Update(); - L2FESpace->Update(); - MEFESpace->Update(); - - const int Vsize_h1 = H1FESpace->GetVSize(); - const int Vsize_l2 = L2FESpace->GetVSize(); - - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - - S_tmp = S; - S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); - - x_gf.MakeRef(H1FESpace, S, true_offset[0]); - v_gf.MakeRef(H1FESpace, S, true_offset[1]); - e_gf.MakeRef(L2FESpace, S, true_offset[2]); - x_gf.SyncAliasMemory(S); - v_gf.SyncAliasMemory(S); - e_gf.SyncAliasMemory(S); - - S_tmp.Update(true_offset); - m_gf.Update(); - - //H1FESpace->UpdatesFinished(); - //L2FESpace->UpdatesFinished(); - //MEFESpace->UpdatesFinished(); -} - -} // amr namesapce - -static void GetZeroBCDofs(ParMesh *pmesh, ParFiniteElementSpace &H1, - const int bdr_attr_max, - Array &ess_tdofs, - Array &ess_vdofs) -{ - ess_tdofs.SetSize(0); - ess_vdofs.SetSize(0); - Array ess_bdr(bdr_attr_max), dofs_marker, dofs_list; - for (int d = 0; d < pmesh->Dimension(); d++) - { - // Attributes 1/2/3 correspond to fixed-x/y/z boundaries, - // i.e., we must enforce v_x/y/z = 0 for the velocity components. - ess_bdr = 0; ess_bdr[d] = 1; - H1.GetEssentialTrueDofs(ess_bdr, dofs_list, d); - ess_tdofs.Append(dofs_list); - H1.GetEssentialVDofs(ess_bdr, dofs_marker, d); - FiniteElementSpace::MarkerToList(dofs_marker, dofs_list); - ess_vdofs.Append(dofs_list); - } -} - -static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements) -{ - Array v; - - for (int i = 0; i < mesh->GetNE(); i++) - { - mesh->GetElementVertices(i, v); - for (int j = 0; j < v.Size(); j++) - { - double dist = 0.0; - for (int l = 0; l < mesh->SpaceDimension(); l++) - { - double d = vert(l) - mesh->GetVertex(v[j])[l]; - dist += d*d; - } - if (dist <= size*size) { elements.Append(i); break; } - } - } -} - -static void Pow(Vector &vec, double p) -{ - for (int i = 0; i < vec.Size(); i++) - { - vec(i) = std::pow(vec(i), p); - } -} - -static void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order = -1) -{ - const FiniteElementSpace *space = gf.FESpace(); - int ne = space->GetNE(); - - if (int_order < 0) { int_order = space->GetOrder(0) + 1; } - - elem_min.SetSize(ne); - elem_max.SetSize(ne); - - Vector vals, tmp; - for (int i = 0; i < ne; i++) - { - int geom = space->GetFE(i)->GetGeomType(); - const IntegrationRule &ir = IntRules.Get(geom, int_order); - - gf.GetValues(i, ir, vals); - - if (space->GetVDim() > 1) - { - Pow(vals, 2.0); - for (int vd = 1; vd < space->GetVDim(); vd++) - { - gf.GetValues(i, ir, tmp, vd+1); - Pow(tmp, 2.0); - vals += tmp; - } - Pow(vals, 0.5); - } - - elem_min(i) = vals.Min(); - elem_max(i) = vals.Max(); - } -} - class HydroODESolver : public ODESolver { protected: From 42dbe0c1d61678274deaa6d389f7d8e80b955f17 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 19 Jan 2021 15:27:34 -0800 Subject: [PATCH 14/22] Added vis_windows --- laghos.cpp | 97 ++++++++++++++++++++++++++++++-------------------- laghos_amr.cpp | 45 +++++++++++------------ laghos_amr.hpp | 44 ++++++++++++++--------- 3 files changed, 107 insertions(+), 79 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 6f3b3686..d41bf9c9 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -74,8 +74,8 @@ using namespace mfem; static long GetMaxRssMB(); -static void Checks(const int, const int, const double, int&); -static void display_banner(std::ostream &os) +static void NonRegressionTests(const int, const int, const double, int&); +static void DisplayBanner(std::ostream &os) { os << std::endl << " __ __ " << std::endl @@ -94,7 +94,7 @@ int main(int argc, char *argv[]) const int myid = mpi.WorldRank(); // Print the banner. - if (mpi.Root()) { display_banner(std::cout); } + if (mpi.Root()) { DisplayBanner(std::cout); } // Parse command-line options. problem = 1; @@ -117,6 +117,7 @@ int main(int argc, char *argv[]) bool impose_visc = false; bool visualization = false; int vis_steps = 5; + int vis_windows = 3; bool visit = false; bool gfprint = false; const char *basename = "results/Laghos"; @@ -180,6 +181,8 @@ int main(int argc, char *argv[]) "Enable or disable GLVis visualization."); args.AddOption(&vis_steps, "-vs", "--visualization-steps", "Visualize every n-th timestep."); + args.AddOption(&vis_windows, "-vw", "--visualization-windows", + "Number of visualization windows to open: 1~3"); args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit", "Enable or disable VisIt visualization."); args.AddOption(&gfprint, "-print", "--print", "-no-print", "--no-print", @@ -230,6 +233,7 @@ int main(int argc, char *argv[]) if (mpi.Root()) { args.PrintOptions(std::cout); } + // Check AMR configuration: only Sedov problem (#1) is supported for now if (amr && problem != 1) { if (mpi.Root()) @@ -305,9 +309,9 @@ int main(int argc, char *argv[]) { dbg("RefineAtVertex BLAST"); mesh->EnsureNCMesh(); + Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; for (int lev = 0; lev < rs_levels; lev++) { - Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; mesh->RefineAtVertex(blast, amr_blast_size); } } @@ -472,7 +476,7 @@ int main(int argc, char *argv[]) GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); // Define the explicit ODE solver used for time integration. - ODESolver *ode_solver = NULL; + ODESolver *ode_solver = nullptr; switch (ode_solver_type) { case 1: ode_solver = new ForwardEulerSolver; break; @@ -589,6 +593,7 @@ int main(int argc, char *argv[]) } if (impose_visc) { visc = true; } + // Time dependent hydro operator hydrodynamics::LagrangianHydroOperator hydro(S.Size(), H1FESpace, L2FESpace, ess_tdofs, @@ -598,18 +603,22 @@ int main(int argc, char *argv[]) p_assembly, amr, cg_tol, cg_max_iter, ftz_tol, order_q); - - amr::Operator *AMR = amr ? new amr::Operator(pmesh, - amr_estimator, - amr_ref_threshold, - amr_jac_threshold, - amr_deref_threshold, - amr_max_level, - amr_blast_size, - amr_nc_limit, - blast_energy, - blast_position) : nullptr; - if (amr) { AMR->Setup(x_gf); } + // AMR operator + amr::Operator *AMR =nullptr; + if (amr) + { + AMR = new amr::Operator(pmesh, + amr_estimator, + amr_ref_threshold, + amr_jac_threshold, + amr_deref_threshold, + amr_max_level, + amr_nc_limit, + amr_blast_size, + blast_energy, + blast_position); + AMR->Setup(x_gf); + } socketstream vis_rho, vis_v, vis_e; char vishost[] = "localhost"; @@ -636,12 +645,18 @@ int main(int argc, char *argv[]) hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); } - Wx += offx; - hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, - "Velocity", Wx, Wy, Ww, Wh); - Wx += offx; - hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, - "Specific Internal Energy", Wx, Wy, Ww, Wh); + if (vis_windows > 1) + { + Wx += offx; + hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, + "Velocity", Wx, Wy, Ww, Wh); + } + if (vis_windows > 2) + { + Wx += offx; + hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, + "Specific Internal Energy", Wx, Wy, Ww, Wh); + } } // Save data for VisIt visualization. @@ -680,6 +695,7 @@ int main(int argc, char *argv[]) t_old = t; hydro.ResetTimeStepEstimate(); + // Reset the associated refiner and derefiner if (amr) { AMR->Reset(); } // S is the vector of dofs, t is the current time, and dt is the time step @@ -760,13 +776,19 @@ int main(int argc, char *argv[]) hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, "Density", Wx, Wy, Ww, Wh); } - Wx += offx; - hydrodynamics::VisualizeField(vis_v, vishost, visport, - v_gf, "Velocity", Wx, Wy, Ww, Wh); - Wx += offx; - hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, - "Specific Internal Energy", - Wx, Wy, Ww,Wh); + if (vis_windows > 1) + { + Wx += offx; + hydrodynamics::VisualizeField(vis_v, vishost, visport, + v_gf, "Velocity", Wx, Wy, Ww, Wh); + } + if (vis_windows > 2) + { + Wx += offx; + hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, + "Specific Internal Energy", + Wx, Wy, Ww,Wh); + } } if (visit) @@ -804,15 +826,13 @@ int main(int argc, char *argv[]) e_gf.SaveAsOne(e_ofs); e_ofs.close(); } - } // last_step + } // last_step or vis_steps + // AMR update if (amr) { - AMR->Update(S, S_old, - x_gf,v_gf,e_gf,m_gf, - true_offset, - hydro, ode_solver, - bdr_attr_max, ess_tdofs, ess_vdofs); + AMR->Update(hydro, ode_solver, S, S_old, x_gf, v_gf, e_gf, m_gf, + true_offset, bdr_attr_max, ess_tdofs, ess_vdofs); } // Problems checks @@ -829,7 +849,7 @@ int main(int argc, char *argv[]) MFEM_VERIFY(cfl==0.5, "check: cfl"); MFEM_VERIFY(strncmp(mesh_file, "default", 7) == 0, "check: mesh_file"); MFEM_VERIFY(dim==2 || dim==3, "check: dimension"); - Checks(dim, ti, e_norm, checks); + NonRegressionTests(dim, ti, e_norm, checks); } } MFEM_VERIFY(!check || checks == 2, "Check error!"); @@ -902,7 +922,8 @@ static bool Check(const double a, const double v, const double eps) return fmax(err_a, err_v) < eps; } -static void Checks(const int dim, const int ti, const double nrm, int &chk) +static void NonRegressionTests(const int dim, const int ti, const double nrm, + int &chk) { const int pb = problem; const double eps = 1.e-13; diff --git a/laghos_amr.cpp b/laghos_amr.cpp index 95753744..1822f1ab 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -131,8 +131,8 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, } } -void Eval(const ParGridFunction &v_gf, const double threshold, - ParMesh &pmesh, Array &refs) +static void Eval(const ParGridFunction &v_gf, const double threshold, + ParMesh &pmesh, Array &refs) { Vector val; IntegrationPoint ip; @@ -167,31 +167,27 @@ Operator::Operator(ParMesh *pmesh, double jac_t, double deref_t, int max_level, - const double blast_size, - const int nc_limit, - const double blast_energy, - const double *blast_xyz): + int nc_limit, + double size_b, + double energy_b, + double *xyz_b): pmesh(pmesh), myid(pmesh->GetMyRank()), dim(pmesh->Dimension()), sdim(pmesh->SpaceDimension()), + flux_fec(order, dim), + flux_fes(pmesh, &flux_fec, sdim), opt( { - estimator, ref_t, jac_t, deref_t, max_level, blast_size, nc_limit, - blast_energy, Vertex(blast_xyz[0], blast_xyz[1], blast_xyz[2]) -}) -{ - dbg(); -} + estimator, ref_t, jac_t, deref_t, max_level, nc_limit, + size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) +}) { dbg(); } Operator::~Operator() { dbg(); } -void Operator::Setup(ParGridFunction &u) +void Operator::Setup(ParGridFunction &x_gf) { - dbg("AMR estimator #%d", opt.estimator); - - flux_fec = new L2_FECollection(order, dim); - auto flux_fes = new ParFiniteElementSpace(pmesh, flux_fec, sdim); + dbg("AMR Setup #%d", opt.estimator); if (opt.estimator == amr::estimator::zz) { @@ -199,7 +195,7 @@ void Operator::Setup(ParGridFunction &u) ei = new amr::EstimatorIntegrator(); smooth_flux_fec = new RT_FECollection(order-1, dim); auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); - estimator = new L2ZienkiewiczZhuEstimator(*ei, u, flux_fes, + estimator = new L2ZienkiewiczZhuEstimator(*ei, x_gf, &flux_fes, smooth_flux_fes); } @@ -207,7 +203,7 @@ void Operator::Setup(ParGridFunction &u) { dbg("Kelly estimator init"); ei = new amr::EstimatorIntegrator(); - estimator = new KellyErrorEstimator(*ei, u, flux_fes); + estimator = new KellyErrorEstimator(*ei, x_gf, flux_fes); } if (estimator) @@ -227,20 +223,19 @@ void Operator::Setup(ParGridFunction &u) void Operator::Reset() { - if (!(refiner && derefiner)) { return; } - refiner->Reset(); - derefiner->Reset(); + if (refiner) { refiner->Reset(); } + if (derefiner) { derefiner->Reset(); } } -void Operator::Update(BlockVector &S, +void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, + ODESolver *ode_solver, + BlockVector &S, BlockVector &S_old, ParGridFunction &x, ParGridFunction &v, ParGridFunction &e, ParGridFunction &m, Array &true_offset, - hydrodynamics::LagrangianHydroOperator &hydro, - ODESolver *ode_solver, const int bdr_attr_max, Array &ess_tdofs, Array &ess_vdofs) diff --git a/laghos_amr.hpp b/laghos_amr.hpp index 5f62e370..31be5ea3 100644 --- a/laghos_amr.hpp +++ b/laghos_amr.hpp @@ -28,6 +28,17 @@ namespace amr enum estimator: int { std = 0, rho = 1, zz = 2, kelly = 3 }; +static const char *Estimator(const int est) +{ + switch (static_cast(est)) + { + case amr::estimator::std: return "Custom Sedov Estimator"; + case amr::estimator::rho: return "RHO Estimator"; + case amr::estimator::zz: return "ZZ Estimator"; + case amr::estimator::kelly: return "Kelly Estimator"; + default: MFEM_ABORT("Unknown estimator!"); + } +} static void Update(BlockVector &S, BlockVector &S_tmp, Array &true_offset, @@ -171,11 +182,22 @@ class EstimatorIntegrator: public DiffusionIntegrator bool with_coef = false); }; +// AMR operator class Operator { const int order = 3; ParMesh *pmesh; const int myid, dim, sdim; + + L2_FECollection flux_fec; + ParFiniteElementSpace flux_fes; + + RT_FECollection *smooth_flux_fec = nullptr; + ErrorEstimator *estimator = nullptr; + ThresholdRefiner *refiner = nullptr; + ThresholdDerefiner *derefiner = nullptr; + amr::EstimatorIntegrator *ei = nullptr; + const struct Options { int estimator; @@ -183,28 +205,18 @@ class Operator double jac_threshold; double deref_threshold; int max_level; - double blast_size; int nc_limit; + double blast_size; double blast_energy; Vertex blast_position; } opt; - // AMR estimator setup - L2_FECollection *flux_fec; - RT_FECollection *smooth_flux_fec; - - ErrorEstimator *estimator = nullptr; - ThresholdRefiner *refiner = nullptr; - ThresholdDerefiner *derefiner = nullptr; - amr::EstimatorIntegrator *ei = nullptr; - public: Operator(ParMesh *pmesh, int estimator, double ref_t, double jac_t, double deref_t, - int max_level, - const double blast_size, const int nc_limit, - const double blast_energy, const double *blast_position); + int max_level, int nc_limit, + double blast_size, double blast_energy, double *blast_position); ~Operator(); @@ -212,15 +224,15 @@ class Operator void Reset(); - void Update(BlockVector &S, + void Update(hydrodynamics::LagrangianHydroOperator &hydro, + ODESolver *ode_solver, + BlockVector &S, BlockVector &S_old, ParGridFunction &x, ParGridFunction &v, ParGridFunction &e, ParGridFunction &m, Array &true_offset, - hydrodynamics::LagrangianHydroOperator&, - ODESolver*, const int bdr_attr_max, Array &ess_tdofs, Array &ess_vdofs); From 429e4f727b62b99df025bb9051c7eb504e8cf9fc Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 19 Jan 2021 19:41:16 -0800 Subject: [PATCH 15/22] WIP derefinement --- laghos.cpp | 18 ++++----- laghos_amr.cpp | 104 +++++++++++++++++++++++++++++-------------------- laghos_amr.hpp | 50 +++++++++++++++++------- 3 files changed, 107 insertions(+), 65 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index d41bf9c9..9a412d6c 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -131,12 +131,12 @@ int main(int argc, char *argv[]) double blast_energy = 0.25; double blast_position[] = {0.0, 0.0, 0.0}; bool amr = false; - int amr_estimator = amr::estimator::std; + int amr_estimator = amr::estimator::custom; double amr_ref_threshold = 2e-4; double amr_jac_threshold = 0.92; double amr_deref_threshold = 0.75; int amr_max_level = rs_levels + rp_levels; - const double amr_blast_size = 1e-10; + const double amr_blast_eps = 1e-10; const int amr_nc_limit = 1; // maximum level of hanging nodes OptionsParser args(argc, argv); @@ -212,14 +212,14 @@ int main(int argc, char *argv[]) args.AddOption(&amr, "-amr", "--enable-amr", "-no-amr", "--disable-amr", "Experimental adaptive mesh refinement (problem 1 only)."); + args.AddOption(&amr_estimator, "-ae", "--amr-estimator", + "AMR estimator: 0:Custom, 1:Rho, 2:ZZ, 3:Kelly"); args.AddOption(&amr_ref_threshold, "-ar", "--amr-ref-threshold", - "AMR refinement threshold."); - args.AddOption(&amr_jac_threshold, "-aj", "--amr-jac-threshold", "AMR Jacobian refinement threshold."); args.AddOption(&amr_deref_threshold, "-ad", "--amr-deref-threshold", + "AMR refinement threshold."); + args.AddOption(&amr_jac_threshold, "-aj", "--amr-jac-threshold", "AMR derefinement threshold (0 = no derefinement)."); - args.AddOption(&amr_estimator, "-ae", "--amr-estimator", - "AMR estimator: 0:vgrad/visc 1:Jacobians"); args.AddOption(&amr_max_level, "-am", "--amr-max-level", "AMR max refined level (default to 'rs_levels + rp_levels')"); args.Parse(); @@ -307,12 +307,12 @@ int main(int argc, char *argv[]) } else { - dbg("RefineAtVertex BLAST"); + dbg("AMR for Sedov problem"); mesh->EnsureNCMesh(); Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; for (int lev = 0; lev < rs_levels; lev++) { - mesh->RefineAtVertex(blast, amr_blast_size); + mesh->RefineAtVertex(blast, amr_blast_eps); } } @@ -614,7 +614,7 @@ int main(int argc, char *argv[]) amr_deref_threshold, amr_max_level, amr_nc_limit, - amr_blast_size, + amr_blast_eps, blast_energy, blast_position); AMR->Setup(x_gf); diff --git a/laghos_amr.cpp b/laghos_amr.cpp index 1822f1ab..d2b822a1 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -26,11 +26,11 @@ namespace mfem namespace amr { -static void ComputeElementFlux0(const FiniteElement &el, - ElementTransformation &Trans, - const Vector &u, - const FiniteElement &fluxelem, - Vector &flux) +void EstimatorIntegrator::ComputeElementFlux1(const FiniteElement &el, + ElementTransformation &Trans, + const Vector &u, + const FiniteElement &fluxelem, + Vector &flux) { const int dof = el.GetDof(); const int dim = el.GetDim(); @@ -61,10 +61,11 @@ static void ComputeElementFlux0(const FiniteElement &el, } } -static void ComputeElementFlux1(const FiniteElement &el, - ElementTransformation &Trans, - const FiniteElement &fluxelem, - Vector &flux) +void EstimatorIntegrator::ComputeElementFlux2(const int e, + const FiniteElement &el, + ElementTransformation &Trans, + const FiniteElement &fluxelem, + Vector &flux) { const int dim = el.GetDim(); const int sdim = Trans.GetSpaceDim(); @@ -79,6 +80,8 @@ static void ComputeElementFlux1(const FiniteElement &el, double minW = +NL_DMAX; double maxW = -NL_DMAX; + const int depth = pmesh->pncmesh->GetElementDepth(e); + for (int q = 0; q < NQ; q++) { const IntegrationPoint &ip = ir.IntPoint(q); @@ -92,11 +95,15 @@ static void ComputeElementFlux1(const FiniteElement &el, MFEM_VERIFY(std::fabs(maxW) > 1e-13, ""); const double rho = minW / maxW; MFEM_VERIFY(rho <= 1.0, ""); - constexpr double amr_jac_threshold = 0.98; for (int d = 0; d < sdim; d++) { - const double value = (rho > amr_jac_threshold) ? 0.0: rho; - flux(NQ*d+q) = value; + const int iq = NQ*d + q; + flux(iq) = 1.0 - rho; + //dbg("%f > %f", rho, jac_threshold); + if (rho > jac_threshold) { continue; } + //dbg("rho:%f, depth:%d", rho, depth); + if (depth > max_level) { continue; } + flux(iq) = rho; } } } @@ -108,6 +115,8 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, Vector &flux, bool with_coef) { + MFEM_VERIFY(NE == pmesh->GetNE(), ""); + //dbg("NE:%d, e:%d", NE, e); // ZZ comes with with_coef set to true, not Kelly switch (flux_mode) { @@ -119,12 +128,12 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, } case mode::one: { - ComputeElementFlux0(el, Trans, u, fluxelem, flux); + ComputeElementFlux1(el, Trans, u, fluxelem, flux); break; } case mode::two: { - ComputeElementFlux1(el, Trans, fluxelem, flux); + ComputeElementFlux2(e++, el, Trans, fluxelem, flux); break; } default: MFEM_ABORT("Unknown mode!"); @@ -181,41 +190,51 @@ Operator::Operator(ParMesh *pmesh, { estimator, ref_t, jac_t, deref_t, max_level, nc_limit, size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) -}) { dbg(); } +}) { dbg("%s", amr::EstimatorName(opt.estimator)); } Operator::~Operator() { dbg(); } void Operator::Setup(ParGridFunction &x_gf) { - dbg("AMR Setup #%d", opt.estimator); + dbg(); + if (myid == 0) + { + std::cout << "AMR setup with " + << amr::EstimatorName(opt.estimator) << " estimator" + << std::endl; + } if (opt.estimator == amr::estimator::zz) { dbg("ZZ estimator init"); - ei = new amr::EstimatorIntegrator(); + integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, + opt.jac_threshold); smooth_flux_fec = new RT_FECollection(order-1, dim); auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); - estimator = new L2ZienkiewiczZhuEstimator(*ei, x_gf, &flux_fes, + estimator = new L2ZienkiewiczZhuEstimator(*integ, x_gf, &flux_fes, smooth_flux_fes); } if (opt.estimator == amr::estimator::kelly) { dbg("Kelly estimator init"); - ei = new amr::EstimatorIntegrator(); - estimator = new KellyErrorEstimator(*ei, x_gf, flux_fes); + integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, + opt.jac_threshold); + estimator = new KellyErrorEstimator(*integ, x_gf, flux_fes); } if (estimator) { - const double max_elem_error = 1.0e-4; + const double hysteresis = 0.25; + const double max_elem_error = 1.0e-6; refiner = new ThresholdRefiner(*estimator); - refiner->SetTotalErrorFraction(0.75); + refiner->SetTotalErrorFraction(0.0); + refiner->SetLocalErrorGoal(max_elem_error); refiner->PreferConformingRefinement(); refiner->SetNCLimit(opt.nc_limit); - const double hysteresis = 0.9; // derefinement safety coefficient derefiner = new ThresholdDerefiner(*estimator); + derefiner->SetOp(2); // 0:min, 1:sum, 2:max derefiner->SetThreshold(hysteresis * max_elem_error); derefiner->SetNCLimit(opt.nc_limit); } @@ -223,6 +242,7 @@ void Operator::Setup(ParGridFunction &x_gf) void Operator::Reset() { + if (integ) { integ->Reset(); } if (refiner) { refiner->Reset(); } if (derefiner) { derefiner->Reset(); } } @@ -251,7 +271,7 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, switch (opt.estimator) { - case amr::estimator::std: + case amr::estimator::custom: { Vector &error_est = hydro.GetZoneMaxVisc(); for (int e = 0; e < NE; e++) @@ -265,8 +285,10 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, } break; } - case amr::estimator::rho: + + case amr::estimator::jjt: { + MFEM_VERIFY(dim == 2, "JJt estimator only available for 2D!"); const double art = opt.ref_threshold; const int order = H1FESpace.GetOrder(0) + 1; DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); @@ -295,6 +317,7 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, { const double rho = minW / maxW; MFEM_VERIFY(rho <= 1.0, ""); + //dbg("%f", rho); if (rho < opt.jac_threshold && depth < opt.max_level) { refs.Append(Refinement(e)); @@ -303,21 +326,21 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, } break; } + case amr::estimator::zz: case amr::estimator::kelly: { - dbg("AMR estimator Apply"); + //dbg("AMR estimator Apply"); refiner->Apply(*pmesh); - if (refiner->Refined()) { mesh_refined = true; } + if (refiner->Refined()) { dbg("\033[1;32mREFINED!"); mesh_refined = true; } MFEM_VERIFY(!refiner->Derefined(),""); MFEM_VERIFY(!refiner->Rebalanced(),""); - /*if (refiner->Stop() && myid == 0) - { std::cout << "Stopping criterion satisfied. Stop.\n"; }*/ break; } default: MFEM_ABORT("Unknown AMR estimator!"); } + // custom and JJt uses refs, ZZ and Kelly will set mesh_refined const int nref = pmesh->ReduceInt(refs.Size()); if (nref && !mesh_refined) { @@ -330,14 +353,13 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, std::cout << "Refined " << nref << " elements." << std::endl; } } - else if (opt.estimator == amr::estimator::std && - opt.deref_threshold >= 0.0) + else if (opt.estimator == amr::estimator::custom && + opt.deref_threshold >= 0.0 && !mesh_refined) { - dbg("STD Derefinement"); + //dbg("Derefinement"); ParGridFunction rho_gf; - hydro.ComputeDensity(rho_gf); - Vector rho_max, rho_min; + hydro.ComputeDensity(rho_gf); GetPerElementMinMax(rho_gf, rho_min, rho_max); // Derefinement based on zone maximum rho in post-shock region @@ -370,26 +392,24 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, std::cout << "Derefined, threshold = " << threshold << std::endl; } } - /*else if ((amr_estimator == amr::estimator::zz || - amr_estimator == amr::estimator::kelly) && - !mesh_refined) + else if ((opt.estimator == amr::estimator::zz || + opt.estimator == amr::estimator::kelly) && !mesh_refined) { - dbg("ZZ/KL Derefinement"); + //dbg("ZZ/Kelly Derefinement"); MFEM_VERIFY(derefiner,""); if (derefiner->Apply(*pmesh)) { if (myid == 0) { - std::cout << "\nDerefined elements." << std::endl; + //std::cout << "\nDerefined elements." << std::endl; } } - if (derefiner->Derefined()) { mesh_refined = true; } - }*/ + if (derefiner->Derefined()) { dbg("\033[1;31mDEREFINED!"); mesh_refined = true; } + } else { /* nothing to do */ } if (mesh_refined) { - dbg("\033[7mmesh_changed"); constexpr bool quick = true; amr::Update(S, S_old, true_offset, x, v, e, m); diff --git a/laghos_amr.hpp b/laghos_amr.hpp index 31be5ea3..8ca26f5e 100644 --- a/laghos_amr.hpp +++ b/laghos_amr.hpp @@ -26,16 +26,16 @@ namespace mfem namespace amr { -enum estimator: int { std = 0, rho = 1, zz = 2, kelly = 3 }; +enum estimator: int { custom = 0, jjt = 1, zz = 2, kelly = 3 }; -static const char *Estimator(const int est) +static const char *EstimatorName(const int est) { switch (static_cast(est)) { - case amr::estimator::std: return "Custom Sedov Estimator"; - case amr::estimator::rho: return "RHO Estimator"; - case amr::estimator::zz: return "ZZ Estimator"; - case amr::estimator::kelly: return "Kelly Estimator"; + case amr::estimator::custom: return "Custom"; + case amr::estimator::jjt: return "JJt"; + case amr::estimator::zz: return "ZZ"; + case amr::estimator::kelly: return "Kelly"; default: MFEM_ABORT("Unknown estimator!"); } } @@ -80,10 +80,6 @@ static void Update(BlockVector &S, BlockVector &S_tmp, S_tmp.Update(true_offset); m_gf.Update(); - - //H1FESpace->UpdatesFinished(); - //L2FESpace->UpdatesFinished(); - //MEFESpace->UpdatesFinished(); } static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, @@ -154,12 +150,26 @@ static void GetPerElementMinMax(const GridFunction &gf, class EstimatorIntegrator: public DiffusionIntegrator { + int NE, e; + ParMesh *pmesh; enum class mode { diffusion, one, two }; const mode flux_mode; ConstantCoefficient one {1.0}; + const int max_level; + const double jac_threshold; public: - EstimatorIntegrator(const mode flux_mode = mode::diffusion): - DiffusionIntegrator(one), flux_mode(flux_mode) { } + EstimatorIntegrator(ParMesh *pmesh, + const int max_level, + const double jac_threshold, + const mode flux_mode = mode::two): + DiffusionIntegrator(one), + NE(pmesh->GetNE()), + pmesh(pmesh), + flux_mode(flux_mode), + max_level(max_level), + jac_threshold(jac_threshold) { } + + void Reset() { e = 0; NE = pmesh->GetNE(); } double ComputeFluxEnergy(const FiniteElement &fluxelem, ElementTransformation &Trans, @@ -180,12 +190,24 @@ class EstimatorIntegrator: public DiffusionIntegrator const FiniteElement &fluxelem, Vector &flux, bool with_coef = false); +private: + void ComputeElementFlux1(const FiniteElement &el, + ElementTransformation &Trans, + const Vector &u, + const FiniteElement &fluxelem, + Vector &flux); + + void ComputeElementFlux2(const int e, + const FiniteElement &el, + ElementTransformation &Trans, + const FiniteElement &fluxelem, + Vector &flux); }; // AMR operator class Operator { - const int order = 3; + const int order = 3; // should be computed ParMesh *pmesh; const int myid, dim, sdim; @@ -196,7 +218,7 @@ class Operator ErrorEstimator *estimator = nullptr; ThresholdRefiner *refiner = nullptr; ThresholdDerefiner *derefiner = nullptr; - amr::EstimatorIntegrator *ei = nullptr; + amr::EstimatorIntegrator *integ = nullptr; const struct Options { From c5e236272370918b5c8aae50d13f5d028338a486 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 20 Jan 2021 10:43:24 -0800 Subject: [PATCH 16/22] Cleanup, readme & runs --- README.md | 9 +++++--- laghos.cpp | 6 ------ laghos_amr.cpp | 55 +++++------------------------------------------ laghos_amr.hpp | 1 + laghos_solver.cpp | 3 ++- 5 files changed, 14 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index b3d0cad3..0d54a574 100644 --- a/README.md +++ b/README.md @@ -257,12 +257,15 @@ The latter produces the following specific internal energy plot (notice the `-vi The AMR version only runs with problem 1 (Sedov blast). New parameters are: - `-amr`: turn on AMR mode -- `-rt` or `--ref-threshold`: tweak the refinement threshold -- `-dt` or `--deref-threshold`: tweak the derefinement threshold +- `-ae` or `--amr-estimator`: available estimators are 0:Custom, 1:Rho, 2:ZZ and 3:Kelly +- `-ar` or `--amr-ref-threshold`: tweak the refinement threshold +- `-ad` or `--amr-deref-threshold`: tweak the derefinement threshold +- `-aj` or `--amr-jac-threshold`: tweak the refinement threshold for the Rho estimator +- `-am` or `--amr-max-level`: twweak the max level of refinement One of the sample runs is: ```sh -mpirun -np 8 laghos -p 1 -m ../data/cube01_hex.mesh -rs 4 -tf 0.6 -rt 1e-3 -amr +mpirun -np 8 laghos -p 1 -dim 3 -rs 4 -amr -tf 0.6 -ar 1e-3 ``` This produces the following plots: diff --git a/laghos.cpp b/laghos.cpp index 9a412d6c..154309e1 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -67,10 +67,6 @@ #include "laghos_amr.hpp" #include "laghos_solver.hpp" -#undef MFEM_DEBUG_COLOR -#define MFEM_DEBUG_COLOR 199 -#include "general/debug.hpp" - using namespace mfem; static long GetMaxRssMB(); @@ -302,12 +298,10 @@ int main(int argc, char *argv[]) // Refine the mesh in serial to increase the resolution. if (!amr) { - dbg("Refine the mesh in serial to increase the resolution"); for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } } else { - dbg("AMR for Sedov problem"); mesh->EnsureNCMesh(); Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; for (int lev = 0; lev < rs_levels; lev++) diff --git a/laghos_amr.cpp b/laghos_amr.cpp index d2b822a1..635167d3 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -14,9 +14,6 @@ // software, applications, hardware, advanced system engineering and early // testbed platforms, in support of the nation's exascale computing imperative. -#define MFEM_DEBUG_COLOR 226 -#include "general/debug.hpp" - #include "laghos_solver.hpp" #include "laghos_amr.hpp" @@ -99,9 +96,7 @@ void EstimatorIntegrator::ComputeElementFlux2(const int e, { const int iq = NQ*d + q; flux(iq) = 1.0 - rho; - //dbg("%f > %f", rho, jac_threshold); if (rho > jac_threshold) { continue; } - //dbg("rho:%f, depth:%d", rho, depth); if (depth > max_level) { continue; } flux(iq) = rho; } @@ -116,7 +111,6 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, bool with_coef) { MFEM_VERIFY(NE == pmesh->GetNE(), ""); - //dbg("NE:%d, e:%d", NE, e); // ZZ comes with with_coef set to true, not Kelly switch (flux_mode) { @@ -140,36 +134,6 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, } } -static void Eval(const ParGridFunction &v_gf, const double threshold, - ParMesh &pmesh, Array &refs) -{ - Vector val; - IntegrationPoint ip; - const int NE = pmesh.GetNE(); - const int dim = pmesh.Dimension(); - const int kmax = dim - 2; - constexpr double w = 1.0; - - //ParFiniteElementSpace *pfes = v_gf.ParFESpace(); - - for (int e = 0; e < NE; e++) - { - for (int x1 = 0; x1 <= 1; x1++) - { - for (int x2 = 0; x2 <= 1; x2++) - { - for (int x3 = 0; x3 <= kmax; x3++) - { - ip.Set(x1, x2, x3, w); - v_gf.GetVectorValue(e, ip, val); - const double l2_norm = val.Norml2(); - if (l2_norm > threshold) { refs.Append(Refinement(e)); } - } - } - } - } -} - Operator::Operator(ParMesh *pmesh, int estimator, double ref_t, @@ -190,13 +154,12 @@ Operator::Operator(ParMesh *pmesh, { estimator, ref_t, jac_t, deref_t, max_level, nc_limit, size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) -}) { dbg("%s", amr::EstimatorName(opt.estimator)); } +}) { } -Operator::~Operator() { dbg(); } +Operator::~Operator() { } void Operator::Setup(ParGridFunction &x_gf) { - dbg(); if (myid == 0) { std::cout << "AMR setup with " @@ -206,7 +169,6 @@ void Operator::Setup(ParGridFunction &x_gf) if (opt.estimator == amr::estimator::zz) { - dbg("ZZ estimator init"); integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, opt.jac_threshold); smooth_flux_fec = new RT_FECollection(order-1, dim); @@ -217,7 +179,6 @@ void Operator::Setup(ParGridFunction &x_gf) if (opt.estimator == amr::estimator::kelly) { - dbg("Kelly estimator init"); integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, opt.jac_threshold); estimator = new KellyErrorEstimator(*integ, x_gf, flux_fes); @@ -288,7 +249,6 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::jjt: { - MFEM_VERIFY(dim == 2, "JJt estimator only available for 2D!"); const double art = opt.ref_threshold; const int order = H1FESpace.GetOrder(0) + 1; DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); @@ -317,7 +277,6 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, { const double rho = minW / maxW; MFEM_VERIFY(rho <= 1.0, ""); - //dbg("%f", rho); if (rho < opt.jac_threshold && depth < opt.max_level) { refs.Append(Refinement(e)); @@ -330,9 +289,8 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::zz: case amr::estimator::kelly: { - //dbg("AMR estimator Apply"); refiner->Apply(*pmesh); - if (refiner->Refined()) { dbg("\033[1;32mREFINED!"); mesh_refined = true; } + if (refiner->Refined()) { mesh_refined = true; } MFEM_VERIFY(!refiner->Derefined(),""); MFEM_VERIFY(!refiner->Rebalanced(),""); break; @@ -344,7 +302,6 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, const int nref = pmesh->ReduceInt(refs.Size()); if (nref && !mesh_refined) { - dbg("GeneralRefinement"); constexpr int non_conforming = 1; pmesh->GeneralRefinement(refs, non_conforming, opt.nc_limit); mesh_refined = true; @@ -356,7 +313,6 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, else if (opt.estimator == amr::estimator::custom && opt.deref_threshold >= 0.0 && !mesh_refined) { - //dbg("Derefinement"); ParGridFunction rho_gf; Vector rho_max, rho_min; hydro.ComputeDensity(rho_gf); @@ -395,16 +351,15 @@ void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, else if ((opt.estimator == amr::estimator::zz || opt.estimator == amr::estimator::kelly) && !mesh_refined) { - //dbg("ZZ/Kelly Derefinement"); MFEM_VERIFY(derefiner,""); if (derefiner->Apply(*pmesh)) { if (myid == 0) { - //std::cout << "\nDerefined elements." << std::endl; + std::cout << "\nDerefined elements." << std::endl; } } - if (derefiner->Derefined()) { dbg("\033[1;31mDEREFINED!"); mesh_refined = true; } + if (derefiner->Derefined()) { mesh_refined = true; } } else { /* nothing to do */ } diff --git a/laghos_amr.hpp b/laghos_amr.hpp index 8ca26f5e..8aaba405 100644 --- a/laghos_amr.hpp +++ b/laghos_amr.hpp @@ -38,6 +38,7 @@ static const char *EstimatorName(const int est) case amr::estimator::kelly: return "Kelly"; default: MFEM_ABORT("Unknown estimator!"); } + return nullptr; } static void Update(BlockVector &S, BlockVector &S_tmp, diff --git a/laghos_solver.cpp b/laghos_solver.cpp index 5df43d5c..be3bfe33 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -844,6 +844,7 @@ void LagrangianHydroOperator::UpdateQuadratureData(const Vector &S) const // This code is only for the 1D/FA mode timer.sw_qdata.Start(); + qdata.rho0DetJ0w.HostRead(); const int nqp = ir.GetNPoints(); ParGridFunction x, v, e; Vector* sptr = const_cast(&S); @@ -1198,7 +1199,7 @@ void QUpdateBody(const int NE, const int e, } } // Track maximum artificial viscosity per zone - // !!! Race !!! + // /!\ Race condition, only used with the custom amr estimator d_el_max_visc[e] = fmax(visc_coeff, d_el_max_visc[e]); d_el_max_vgrad[e] = fmax(fabs(det_v_grad), d_el_max_vgrad[e]); } From 2f7fcc5f72a3a5d1010e6f3280ca4604fcdb6129 Mon Sep 17 00:00:00 2001 From: camierjs Date: Mon, 25 Jan 2021 07:57:11 -0800 Subject: [PATCH 17/22] Sync with master --- laghos.cpp | 2 +- laghos_solver.cpp | 71 ----------------------------------------------- laghos_solver.hpp | 7 +++-- 3 files changed, 6 insertions(+), 74 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index c375d390..69ec7a53 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -826,7 +826,7 @@ int main(int argc, char *argv[]) if (amr) { AMR->Update(hydro, ode_solver, S, S_old, x_gf, v_gf, e_gf, m_gf, - true_offset, bdr_attr_max, ess_tdofs, ess_vdofs); + offset, bdr_attr_max, ess_tdofs, ess_vdofs); } // Problems checks diff --git a/laghos_solver.cpp b/laghos_solver.cpp index 7f93476c..9f133b78 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -681,77 +681,6 @@ void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) const } } -double ComputeVolumeIntegral(const int DIM, const int NE,const int NQ, - const int Q1D,const int VDIM,const double ln_norm, - const mfem::Vector& mass, const mfem::Vector& f) -{ - - auto f_vals = mfem::Reshape(f.Read(),VDIM,NQ, NE); - mfem::Vector integrand(NE*NQ); - auto I = Reshape(integrand.Write(), NQ, NE); - - if (DIM == 1) - { - for (int e=0; e < NE; ++e) - { - for (int q = 0; q < NQ; ++q) - { - double vmag = 0; - for (int k = 0; k < VDIM; k++) - { - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - } - else if (DIM == 2) - { - MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1, - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + qy * Q1D; - double vmag = 0; - for (int k = 0; k < VDIM; k++) - { - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - }); - } - else if (DIM == 3) - { - MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D, - { - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const int q = qx + (qy + qz * Q1D) * Q1D; - double vmag = 0; - for (int k = 0; k < VDIM; k++) - { - vmag += pow(f_vals(k,q,e),ln_norm); - } - I(q,e) = vmag; - } - } - } - }); - - } - const double integral = integrand * mass; - return integral; - -} - double LagrangianHydroOperator::InternalEnergy(const ParGridFunction &gf) const { double glob_ie = 0.0; diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 4b372a5e..30fcceaf 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -26,7 +26,7 @@ namespace mfem { // Choice for the problem setup, statically used in functions below. -static int problem; + static int problem, dim; static double gamma_func(const Vector &x) { @@ -51,7 +51,10 @@ static double rho0(const Vector &x) case 0: return 1.0; case 1: return 1.0; case 2: return (x(0) < 0.5) ? 1.0 : 0.1; - case 3: return (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0; + case 3: return (dim == 2) ? (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0 + : x(0) > 1.0 && ((x(1) < 1.5 && x(2) < 1.5) || + (x(1) > 1.5 && x(2) > 1.5)) ? 0.125 : 1.0; + case 4: return 1.0; case 5: { From 7b31d582ee50fcd38c00c801665a6267e641cba3 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 24 May 2022 17:45:52 -0700 Subject: [PATCH 18/22] Cleanup --- README.md | 2 +- laghos.cpp | 34 ++++---- laghos_amr.cpp | 214 ++++++++++++++++++++++++++++++++++++---------- laghos_amr.hpp | 154 ++++----------------------------- laghos_solver.hpp | 11 ++- makefile | 2 +- 6 files changed, 208 insertions(+), 209 deletions(-) diff --git a/README.md b/README.md index c2c489a7..34abf116 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Other computational motives in Laghos include the following: partially assembled) and is applied just twice per "assembly". Both the preparation and the application costs are important for this operator. - Domain-decomposed MPI parallelism. -- Automatic mesh refinement (AMR) with parallel partitioning and load balancing +- Adaptive mesh refinement (AMR) with parallel partitioning and load balancing based on MFEM's non-conforming mesh algorithm that partitions a space-filling curve. Only the Sedov problem (#1) is supported. - Optional in-situ visualization with [GLVis](http:/glvis.org) and data output diff --git a/laghos.cpp b/laghos.cpp index d9e7327d..d758c9bf 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -70,7 +70,7 @@ using namespace mfem; static long GetMaxRssMB(); -static void NonRegressionTests(const int, const int, const double, int&); +static void NonRegressionTests(const int, const double, int&); static void DisplayBanner(std::ostream &os) { os << std::endl @@ -600,19 +600,19 @@ int main(int argc, char *argv[]) cg_tol, cg_max_iter, ftz_tol, order_q); // AMR operator - amr::Operator *AMR =nullptr; + amr::AMR *AMR = nullptr; if (amr) { - AMR = new amr::Operator(pmesh, - amr_estimator, - amr_ref_threshold, - amr_jac_threshold, - amr_deref_threshold, - amr_max_level, - amr_nc_limit, - amr_blast_eps, - blast_energy, - blast_position); + AMR = new amr::AMR(pmesh, + amr_estimator, + amr_ref_threshold, + amr_jac_threshold, + amr_deref_threshold, + amr_max_level, + amr_nc_limit, + amr_blast_eps, + blast_energy, + blast_position); AMR->Setup(x_gf); } @@ -845,7 +845,7 @@ int main(int argc, char *argv[]) MFEM_VERIFY(cfl==0.5, "check: cfl"); MFEM_VERIFY(strncmp(mesh_file, "default", 7) == 0, "check: mesh_file"); MFEM_VERIFY(dim==2 || dim==3, "check: dimension"); - NonRegressionTests(dim, ti, e_norm, checks); + NonRegressionTests(ti, e_norm, checks); } } MFEM_VERIFY(!check || checks == 2, "Check error!"); @@ -918,13 +918,12 @@ static bool Check(const double a, const double v, const double eps) return fmax(err_a, err_v) < eps; } -static void NonRegressionTests(const int dim, const int ti, const double nrm, - int &chk) +static void NonRegressionTests(const int ti, const double nrm, int &chk) { const int pb = problem; const double eps = 1.e-13; printf("%.15e\n",nrm); - if (dim==2) + if (dim == 2) { constexpr double p0_05 = 6.54653862453438e+00; constexpr double p0_27 = 7.58857635779292e+00; @@ -959,7 +958,8 @@ static void NonRegressionTests(const int dim, const int ti, const double nrm, if (pb==7 && ti==05) {chk++; MFEM_VERIFY(Check(nrm,p7_05,eps),"P7, #05");} if (pb==7 && ti==25) {chk++; MFEM_VERIFY(Check(nrm,p7_25,eps),"P7, #25");} } - if (dim==3) + + if (dim == 3) { constexpr double p0_05 = 1.198510951452527e+03; constexpr double p0_188 = 1.199384410059154e+03; diff --git a/laghos_amr.cpp b/laghos_amr.cpp index 635167d3..38e9500b 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -23,11 +23,90 @@ namespace mfem namespace amr { -void EstimatorIntegrator::ComputeElementFlux1(const FiniteElement &el, - ElementTransformation &Trans, - const Vector &u, - const FiniteElement &fluxelem, - Vector &flux) +static const char *EstimatorName(const int est) +{ + switch (static_cast(est)) + { + case amr::estimator::custom: return "Custom"; + case amr::estimator::jjt: return "JJt"; + case amr::estimator::zz: return "ZZ"; + case amr::estimator::kelly: return "Kelly"; + default: MFEM_ABORT("Unknown estimator!"); + } + return nullptr; +} + +static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, + const double size, Array &elements) +{ + Array v; + + for (int i = 0; i < mesh->GetNE(); i++) + { + mesh->GetElementVertices(i, v); + for (int j = 0; j < v.Size(); j++) + { + double dist = 0.0; + for (int l = 0; l < mesh->SpaceDimension(); l++) + { + double d = vert(l) - mesh->GetVertex(v[j])[l]; + dist += d*d; + } + if (dist <= size*size) { elements.Append(i); break; } + } + } +} + +static void Pow(Vector &vec, double p) +{ + for (int i = 0; i < vec.Size(); i++) + { + vec(i) = std::pow(vec(i), p); + } +} + +static void GetPerElementMinMax(const GridFunction &gf, + Vector &elem_min, Vector &elem_max, + int int_order = -1) +{ + const FiniteElementSpace *space = gf.FESpace(); + int ne = space->GetNE(); + + if (int_order < 0) { int_order = space->GetOrder(0) + 1; } + + elem_min.SetSize(ne); + elem_max.SetSize(ne); + + Vector vals, tmp; + for (int i = 0; i < ne; i++) + { + int geom = space->GetFE(i)->GetGeomType(); + const IntegrationRule &ir = IntRules.Get(geom, int_order); + + gf.GetValues(i, ir, vals); + + if (space->GetVDim() > 1) + { + Pow(vals, 2.0); + for (int vd = 1; vd < space->GetVDim(); vd++) + { + gf.GetValues(i, ir, tmp, vd+1); + Pow(tmp, 2.0); + vals += tmp; + } + Pow(vals, 0.5); + } + + elem_min(i) = vals.Min(); + elem_max(i) = vals.Max(); + } +} + +void AMREstimatorIntegrator::ComputeElementFlux1(const FiniteElement &el, + ElementTransformation &Trans, + const Vector &u, + const FiniteElement &fluxelem, + Vector &flux) { const int dof = el.GetDof(); const int dim = el.GetDim(); @@ -58,11 +137,11 @@ void EstimatorIntegrator::ComputeElementFlux1(const FiniteElement &el, } } -void EstimatorIntegrator::ComputeElementFlux2(const int e, - const FiniteElement &el, - ElementTransformation &Trans, - const FiniteElement &fluxelem, - Vector &flux) +void AMREstimatorIntegrator::ComputeElementFlux2(const int e, + const FiniteElement &el, + ElementTransformation &Trans, + const FiniteElement &fluxelem, + Vector &flux) { const int dim = el.GetDim(); const int sdim = Trans.GetSpaceDim(); @@ -103,13 +182,15 @@ void EstimatorIntegrator::ComputeElementFlux2(const int e, } } -void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, - ElementTransformation &Trans, - Vector &u, - const FiniteElement &fluxelem, - Vector &flux, - bool with_coef) +void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, + ElementTransformation &Trans, + Vector &u, + const FiniteElement &fluxelem, + Vector &flux, + bool with_coef, + const IntegrationRule *ir) { + MFEM_VERIFY(ir == NULL, "ir not supported"); MFEM_VERIFY(NE == pmesh->GetNE(), ""); // ZZ comes with with_coef set to true, not Kelly switch (flux_mode) @@ -134,16 +215,16 @@ void EstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, } } -Operator::Operator(ParMesh *pmesh, - int estimator, - double ref_t, - double jac_t, - double deref_t, - int max_level, - int nc_limit, - double size_b, - double energy_b, - double *xyz_b): +AMR::AMR(ParMesh *pmesh, + int estimator, + double ref_t, + double jac_t, + double deref_t, + int max_level, + int nc_limit, + double size_b, + double energy_b, + double *xyz_b): pmesh(pmesh), myid(pmesh->GetMyRank()), dim(pmesh->Dimension()), @@ -153,12 +234,12 @@ Operator::Operator(ParMesh *pmesh, opt( { estimator, ref_t, jac_t, deref_t, max_level, nc_limit, - size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) + size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) }) { } -Operator::~Operator() { } +AMR::~AMR() { } -void Operator::Setup(ParGridFunction &x_gf) +void AMR::Setup(ParGridFunction &x_gf) { if (myid == 0) { @@ -169,8 +250,8 @@ void Operator::Setup(ParGridFunction &x_gf) if (opt.estimator == amr::estimator::zz) { - integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, - opt.jac_threshold); + integ = new AMREstimatorIntegrator(pmesh, opt.max_level, + opt.jac_threshold); smooth_flux_fec = new RT_FECollection(order-1, dim); auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); estimator = new L2ZienkiewiczZhuEstimator(*integ, x_gf, &flux_fes, @@ -179,8 +260,7 @@ void Operator::Setup(ParGridFunction &x_gf) if (opt.estimator == amr::estimator::kelly) { - integ = new amr::EstimatorIntegrator(pmesh, opt.max_level, - opt.jac_threshold); + integ = new AMREstimatorIntegrator(pmesh, opt.max_level, opt.jac_threshold); estimator = new KellyErrorEstimator(*integ, x_gf, flux_fes); } @@ -201,25 +281,67 @@ void Operator::Setup(ParGridFunction &x_gf) } } -void Operator::Reset() +void AMR::Reset() { if (integ) { integ->Reset(); } if (refiner) { refiner->Reset(); } if (derefiner) { derefiner->Reset(); } } -void Operator::Update(hydrodynamics::LagrangianHydroOperator &hydro, - ODESolver *ode_solver, - BlockVector &S, - BlockVector &S_old, - ParGridFunction &x, - ParGridFunction &v, - ParGridFunction &e, - ParGridFunction &m, - Array &true_offset, - const int bdr_attr_max, - Array &ess_tdofs, - Array &ess_vdofs) +static void Update(BlockVector &S, BlockVector &S_tmp, + Array &true_offset, + ParGridFunction &x_gf, + ParGridFunction &v_gf, + ParGridFunction &e_gf, + ParGridFunction &m_gf) +{ + ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); + ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); + ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); + + H1FESpace->Update(); + L2FESpace->Update(); + MEFESpace->Update(); + + const int Vsize_h1 = H1FESpace->GetVSize(); + const int Vsize_l2 = L2FESpace->GetVSize(); + + true_offset[0] = 0; + true_offset[1] = true_offset[0] + Vsize_h1; + true_offset[2] = true_offset[1] + Vsize_h1; + true_offset[3] = true_offset[2] + Vsize_l2; + + S_tmp = S; + S.Update(true_offset); + const Operator* H1Update = H1FESpace->GetUpdateOperator(); + const Operator* L2Update = L2FESpace->GetUpdateOperator(); + H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); + H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); + L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); + + x_gf.MakeRef(H1FESpace, S, true_offset[0]); + v_gf.MakeRef(H1FESpace, S, true_offset[1]); + e_gf.MakeRef(L2FESpace, S, true_offset[2]); + x_gf.SyncAliasMemory(S); + v_gf.SyncAliasMemory(S); + e_gf.SyncAliasMemory(S); + + S_tmp.Update(true_offset); + m_gf.Update(); +} + +void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, + ODESolver *ode_solver, + BlockVector &S, + BlockVector &S_old, + ParGridFunction &x, + ParGridFunction &v, + ParGridFunction &e, + ParGridFunction &m, + Array &true_offset, + const int bdr_attr_max, + Array &ess_tdofs, + Array &ess_vdofs) { Vector v_max, v_min; Array refs; diff --git a/laghos_amr.hpp b/laghos_amr.hpp index 8aaba405..a3ec34af 100644 --- a/laghos_amr.hpp +++ b/laghos_amr.hpp @@ -28,128 +28,7 @@ namespace amr enum estimator: int { custom = 0, jjt = 1, zz = 2, kelly = 3 }; -static const char *EstimatorName(const int est) -{ - switch (static_cast(est)) - { - case amr::estimator::custom: return "Custom"; - case amr::estimator::jjt: return "JJt"; - case amr::estimator::zz: return "ZZ"; - case amr::estimator::kelly: return "Kelly"; - default: MFEM_ABORT("Unknown estimator!"); - } - return nullptr; -} - -static void Update(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf, - ParGridFunction &m_gf) -{ - ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); - ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); - - H1FESpace->Update(); - L2FESpace->Update(); - MEFESpace->Update(); - - const int Vsize_h1 = H1FESpace->GetVSize(); - const int Vsize_l2 = L2FESpace->GetVSize(); - - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - - S_tmp = S; - S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); - - x_gf.MakeRef(H1FESpace, S, true_offset[0]); - v_gf.MakeRef(H1FESpace, S, true_offset[1]); - e_gf.MakeRef(L2FESpace, S, true_offset[2]); - x_gf.SyncAliasMemory(S); - v_gf.SyncAliasMemory(S); - e_gf.SyncAliasMemory(S); - - S_tmp.Update(true_offset); - m_gf.Update(); -} - -static void FindElementsWithVertex(const Mesh* mesh, const Vertex &vert, - const double size, Array &elements) -{ - Array v; - - for (int i = 0; i < mesh->GetNE(); i++) - { - mesh->GetElementVertices(i, v); - for (int j = 0; j < v.Size(); j++) - { - double dist = 0.0; - for (int l = 0; l < mesh->SpaceDimension(); l++) - { - double d = vert(l) - mesh->GetVertex(v[j])[l]; - dist += d*d; - } - if (dist <= size*size) { elements.Append(i); break; } - } - } -} - -static void Pow(Vector &vec, double p) -{ - for (int i = 0; i < vec.Size(); i++) - { - vec(i) = std::pow(vec(i), p); - } -} - -static void GetPerElementMinMax(const GridFunction &gf, - Vector &elem_min, Vector &elem_max, - int int_order = -1) -{ - const FiniteElementSpace *space = gf.FESpace(); - int ne = space->GetNE(); - - if (int_order < 0) { int_order = space->GetOrder(0) + 1; } - - elem_min.SetSize(ne); - elem_max.SetSize(ne); - - Vector vals, tmp; - for (int i = 0; i < ne; i++) - { - int geom = space->GetFE(i)->GetGeomType(); - const IntegrationRule &ir = IntRules.Get(geom, int_order); - - gf.GetValues(i, ir, vals); - - if (space->GetVDim() > 1) - { - Pow(vals, 2.0); - for (int vd = 1; vd < space->GetVDim(); vd++) - { - gf.GetValues(i, ir, tmp, vd+1); - Pow(tmp, 2.0); - vals += tmp; - } - Pow(vals, 0.5); - } - - elem_min(i) = vals.Min(); - elem_max(i) = vals.Max(); - } -} - -class EstimatorIntegrator: public DiffusionIntegrator +class AMREstimatorIntegrator: public DiffusionIntegrator { int NE, e; ParMesh *pmesh; @@ -159,10 +38,10 @@ class EstimatorIntegrator: public DiffusionIntegrator const int max_level; const double jac_threshold; public: - EstimatorIntegrator(ParMesh *pmesh, - const int max_level, - const double jac_threshold, - const mode flux_mode = mode::two): + AMREstimatorIntegrator(ParMesh *pmesh, + const int max_level, + const double jac_threshold, + const mode flux_mode = mode::two): DiffusionIntegrator(one), NE(pmesh->GetNE()), pmesh(pmesh), @@ -187,10 +66,9 @@ class EstimatorIntegrator: public DiffusionIntegrator virtual void ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans, - Vector &u, - const FiniteElement &fluxelem, - Vector &flux, - bool with_coef = false); + Vector &u, const FiniteElement &fluxelem, + Vector &flux, bool with_coef = true, + const IntegrationRule *ir = NULL); private: void ComputeElementFlux1(const FiniteElement &el, ElementTransformation &Trans, @@ -206,7 +84,7 @@ class EstimatorIntegrator: public DiffusionIntegrator }; // AMR operator -class Operator +class AMR { const int order = 3; // should be computed ParMesh *pmesh; @@ -219,7 +97,7 @@ class Operator ErrorEstimator *estimator = nullptr; ThresholdRefiner *refiner = nullptr; ThresholdDerefiner *derefiner = nullptr; - amr::EstimatorIntegrator *integ = nullptr; + AMREstimatorIntegrator *integ = nullptr; const struct Options { @@ -235,13 +113,13 @@ class Operator } opt; public: - Operator(ParMesh *pmesh, - int estimator, - double ref_t, double jac_t, double deref_t, - int max_level, int nc_limit, - double blast_size, double blast_energy, double *blast_position); + AMR(ParMesh *pmesh, + int estimator, + double ref_t, double jac_t, double deref_t, + int max_level, int nc_limit, + double blast_size, double blast_energy, double *blast_position); - ~Operator(); + ~AMR(); void Setup(ParGridFunction&); diff --git a/laghos_solver.hpp b/laghos_solver.hpp index 30fcceaf..bba9891f 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -26,7 +26,7 @@ namespace mfem { // Choice for the problem setup, statically used in functions below. - static int problem, dim; +static int problem, dim; static double gamma_func(const Vector &x) { @@ -51,9 +51,9 @@ static double rho0(const Vector &x) case 0: return 1.0; case 1: return 1.0; case 2: return (x(0) < 0.5) ? 1.0 : 0.1; - case 3: return (dim == 2) ? (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0 - : x(0) > 1.0 && ((x(1) < 1.5 && x(2) < 1.5) || - (x(1) > 1.5 && x(2) > 1.5)) ? 0.125 : 1.0; + case 3: return (dim == 2) ? (x(0) > 1.0 && x(1) > 1.5) ? 0.125 : 1.0 + : x(0) > 1.0 && ((x(1) < 1.5 && x(2) < 1.5) || + (x(1) > 1.5 && x(2) > 1.5)) ? 0.125 : 1.0; case 4: return 1.0; case 5: @@ -449,8 +449,7 @@ class RTCoefficient : public VectorCoefficient public: RTCoefficient(int dim) : VectorCoefficient(dim) { } using VectorCoefficient::Eval; - virtual void Eval(Vector &V, ElementTransformation &T, - const IntegrationPoint &ip) + virtual void Eval(Vector &V, ElementTransformation&, const IntegrationPoint&) { V = 0.0; V(1) = -1.0; } diff --git a/makefile b/makefile index a59f8443..07a9fae1 100644 --- a/makefile +++ b/makefile @@ -111,7 +111,7 @@ OBJECT_FILES = $(SOURCE_FILES:.cpp=.o) laghos: $(OBJECT_FILES) $(CONFIG_MK) $(MFEM_LIB_FILE) $(MFEM_CXX) $(MFEM_LINK_FLAGS) -o laghos $(OBJECT_FILES) $(LIBS) -all:;@$(MAKE) -j $(NPROC) laghos +all: laghos $(OBJECT_FILES): $(HEADER_FILES) $(CONFIG_MK) From 3eb016515d9d7b38acf89ff62fb7b41fbd928ad7 Mon Sep 17 00:00:00 2001 From: camierjs Date: Tue, 24 May 2022 18:15:00 -0700 Subject: [PATCH 19/22] Update tests with latest mfem --- makefile | 2 +- serial/makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index 07a9fae1..da69c408 100644 --- a/makefile +++ b/makefile @@ -236,7 +236,7 @@ tests: cat RUN.dat | tail -n 21 | head -n 1 | \ awk '{ printf("step = %04d, dt = %s |e| = %.10e\n", $$2, $$8, $$11); }' >> RESULTS.dat $(shell cat << EOF > BASELINE.dat) - $(shell echo 'step = 0339, dt = 0.000702, |e| = 4.9695537349e+01' >> BASELINE.dat) + $(shell echo 'step = 0339, dt = 0.000702, |e| = 4.9692294385e+01' >> BASELINE.dat) $(shell echo 'step = 1041, dt = 0.000121, |e| = 3.3909635545e+03' >> BASELINE.dat) $(shell echo 'step = 1154, dt = 0.001655, |e| = 4.6303396053e+01' >> BASELINE.dat) $(shell echo 'step = 0560, dt = 0.002449, |e| = 1.3408616722e+02' >> BASELINE.dat) diff --git a/serial/makefile b/serial/makefile index 9fcf3766..0c4a183b 100644 --- a/serial/makefile +++ b/serial/makefile @@ -219,7 +219,7 @@ tests: cat RUN.dat | tail -n 20 | head -n 1 | \ awk '{ printf("step = %04d, dt = %s |e| = %.10e\n", $$2, $$8, $$11); }' >> RESULTS.dat $(shell cat << EOF > BASELINE.dat) - $(shell echo 'step = 0339, dt = 0.000702, |e| = 4.9695537349e+01' >> BASELINE.dat) + $(shell echo 'step = 0339, dt = 0.000702, |e| = 4.9692294385e+01' >> BASELINE.dat) $(shell echo 'step = 1041, dt = 0.000121, |e| = 3.3909635545e+03' >> BASELINE.dat) $(shell echo 'step = 1154, dt = 0.001655, |e| = 4.6303396053e+01' >> BASELINE.dat) $(shell echo 'step = 0560, dt = 0.002449, |e| = 1.3408616722e+02' >> BASELINE.dat) From 15f358176dbfc23bba8d9ab6bcb5df99886b5874 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 22 Jun 2022 12:13:01 -0700 Subject: [PATCH 20/22] Cleanup shadowed variables --- laghos_amr.cpp | 146 ++++++++++++++++++++++--------------------------- 1 file changed, 66 insertions(+), 80 deletions(-) diff --git a/laghos_amr.cpp b/laghos_amr.cpp index 38e9500b..c6663133 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -137,13 +137,13 @@ void AMREstimatorIntegrator::ComputeElementFlux1(const FiniteElement &el, } } -void AMREstimatorIntegrator::ComputeElementFlux2(const int e, - const FiniteElement &el, +void AMREstimatorIntegrator::ComputeElementFlux2(const int el, + const FiniteElement &fe, ElementTransformation &Trans, const FiniteElement &fluxelem, Vector &flux) { - const int dim = el.GetDim(); + const int dim = fe.GetDim(); const int sdim = Trans.GetSpaceDim(); DenseMatrix Jadjt(dim, sdim), Jadj(dim, sdim); @@ -156,7 +156,7 @@ void AMREstimatorIntegrator::ComputeElementFlux2(const int e, double minW = +NL_DMAX; double maxW = -NL_DMAX; - const int depth = pmesh->pncmesh->GetElementDepth(e); + const int depth = pmesh->pncmesh->GetElementDepth(el); for (int q = 0; q < NQ; q++) { @@ -182,8 +182,8 @@ void AMREstimatorIntegrator::ComputeElementFlux2(const int e, } } -void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, - ElementTransformation &Trans, +void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &fe, + ElementTransformation &Tr, Vector &u, const FiniteElement &fluxelem, Vector &flux, @@ -197,34 +197,28 @@ void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &el, { case mode::diffusion: { - DiffusionIntegrator::ComputeElementFlux(el, Trans, u, + DiffusionIntegrator::ComputeElementFlux(fe, Tr, u, fluxelem, flux, with_coef); break; } case mode::one: { - ComputeElementFlux1(el, Trans, u, fluxelem, flux); + AMREstimatorIntegrator::ComputeElementFlux1(fe, Tr, u, fluxelem, flux); break; } case mode::two: { - ComputeElementFlux2(e++, el, Trans, fluxelem, flux); + AMREstimatorIntegrator::ComputeElementFlux2(e++, fe, Tr, fluxelem, flux); break; } default: MFEM_ABORT("Unknown mode!"); } } -AMR::AMR(ParMesh *pmesh, - int estimator, - double ref_t, - double jac_t, - double deref_t, - int max_level, - int nc_limit, - double size_b, - double energy_b, - double *xyz_b): +AMR::AMR(ParMesh *pmesh, int estimator, + double ref_t, double jac_t, double deref_t, + int max_level, int nc_limit, + double size_b, double energy_b, double *xyz_b): pmesh(pmesh), myid(pmesh->GetMyRank()), dim(pmesh->Dimension()), @@ -288,48 +282,6 @@ void AMR::Reset() if (derefiner) { derefiner->Reset(); } } -static void Update(BlockVector &S, BlockVector &S_tmp, - Array &true_offset, - ParGridFunction &x_gf, - ParGridFunction &v_gf, - ParGridFunction &e_gf, - ParGridFunction &m_gf) -{ - ParFiniteElementSpace* H1FESpace = x_gf.ParFESpace(); - ParFiniteElementSpace* L2FESpace = e_gf.ParFESpace(); - ParFiniteElementSpace* MEFESpace = m_gf.ParFESpace(); - - H1FESpace->Update(); - L2FESpace->Update(); - MEFESpace->Update(); - - const int Vsize_h1 = H1FESpace->GetVSize(); - const int Vsize_l2 = L2FESpace->GetVSize(); - - true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; - - S_tmp = S; - S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_tmp.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_tmp.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_tmp.GetBlock(2), S.GetBlock(2)); - - x_gf.MakeRef(H1FESpace, S, true_offset[0]); - v_gf.MakeRef(H1FESpace, S, true_offset[1]); - e_gf.MakeRef(L2FESpace, S, true_offset[2]); - x_gf.SyncAliasMemory(S); - v_gf.SyncAliasMemory(S); - e_gf.SyncAliasMemory(S); - - S_tmp.Update(true_offset); - m_gf.Update(); -} - void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, ODESolver *ode_solver, BlockVector &S, @@ -357,13 +309,13 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::custom: { Vector &error_est = hydro.GetZoneMaxVisc(); - for (int e = 0; e < NE; e++) + for (int el = 0; el < NE; el++) { - if (error_est(e) > opt.ref_threshold && - pmesh->pncmesh->GetElementDepth(e) < opt.max_level && - (v_min(e) < 1e-3)) // only refine the still area + if (error_est(el) > opt.ref_threshold && + pmesh->pncmesh->GetElementDepth(el) < opt.max_level && + (v_min(el) < 1e-3)) // only refine the still area { - refs.Append(Refinement(e)); + refs.Append(Refinement(el)); } } break; @@ -372,17 +324,17 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::jjt: { const double art = opt.ref_threshold; - const int order = H1FESpace.GetOrder(0) + 1; + const int h1_order = H1FESpace.GetOrder(0) + 1; DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); MFEM_VERIFY(art >= 0.0, "AMR threshold should be positive"); - for (int e = 0; e < NE; e++) + for (int el = 0; el < NE; el++) { double minW = +NL_DMAX; double maxW = -NL_DMAX; - const int depth = pmesh->pncmesh->GetElementDepth(e); - ElementTransformation *eTr = pmesh->GetElementTransformation(e); - const Geometry::Type &type = pmesh->GetElement(e)->GetGeometryType(); - const IntegrationRule *ir = &IntRules.Get(type, order); + const int depth = pmesh->pncmesh->GetElementDepth(el); + ElementTransformation *eTr = pmesh->GetElementTransformation(el); + const Geometry::Type &type = pmesh->GetElement(el)->GetGeometryType(); + const IntegrationRule *ir = &IntRules.Get(type, h1_order); const int NQ = ir->GetNPoints(); for (int q = 0; q < NQ; q++) { @@ -401,7 +353,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, MFEM_VERIFY(rho <= 1.0, ""); if (rho < opt.jac_threshold && depth < opt.max_level) { - refs.Append(Refinement(e)); + refs.Append(Refinement(el)); } } } @@ -476,10 +428,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, MFEM_VERIFY(derefiner,""); if (derefiner->Apply(*pmesh)) { - if (myid == 0) - { - std::cout << "\nDerefined elements." << std::endl; - } + //if (myid == 0) { std::cout << "\nDerefined elements." << std::endl; } } if (derefiner->Derefined()) { mesh_refined = true; } } @@ -487,15 +436,52 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, if (mesh_refined) { + auto Update = [&]() + { + ParFiniteElementSpace* H1FESpace = x.ParFESpace(); + ParFiniteElementSpace* L2FESpace = e.ParFESpace(); + ParFiniteElementSpace* MEFESpace = m.ParFESpace(); + + H1FESpace->Update(); + L2FESpace->Update(); + MEFESpace->Update(); + + const int Vsize_h1 = H1FESpace->GetVSize(); + const int Vsize_l2 = L2FESpace->GetVSize(); + + true_offset[0] = 0; + true_offset[1] = true_offset[0] + Vsize_h1; + true_offset[2] = true_offset[1] + Vsize_h1; + true_offset[3] = true_offset[2] + Vsize_l2; + + S_old = S; + S.Update(true_offset); + const Operator* H1Update = H1FESpace->GetUpdateOperator(); + const Operator* L2Update = L2FESpace->GetUpdateOperator(); + H1Update->Mult(S_old.GetBlock(0), S.GetBlock(0)); + H1Update->Mult(S_old.GetBlock(1), S.GetBlock(1)); + L2Update->Mult(S_old.GetBlock(2), S.GetBlock(2)); + + x.MakeRef(H1FESpace, S, true_offset[0]); + v.MakeRef(H1FESpace, S, true_offset[1]); + e.MakeRef(L2FESpace, S, true_offset[2]); + x.SyncAliasMemory(S); + v.SyncAliasMemory(S); + e.SyncAliasMemory(S); + + S_old.Update(true_offset); + m.Update(); + }; + constexpr bool quick = true; - amr::Update(S, S_old, true_offset, x, v, e, m); + Update(); hydro.AMRUpdate(S, quick); pmesh->Rebalance(); - amr::Update(S, S_old, true_offset, x, v, e, m); - hydro.AMRUpdate(S, !quick); + Update(); + hydro.AMRUpdate(S, not quick); GetZeroBCDofs(pmesh, H1FESpace, bdr_attr_max, ess_tdofs, ess_vdofs); ode_solver->Init(hydro); From 054bcbf38a8c32e0c4bbfdbef91898ea913affb8 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 22 Jun 2022 17:48:34 -0700 Subject: [PATCH 21/22] Cleanup & documentation --- laghos_amr.cpp | 70 ++++++++++++++++++++++------------------------- laghos_amr.hpp | 9 +++++- laghos_solver.cpp | 16 +++++------ 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/laghos_amr.cpp b/laghos_amr.cpp index c6663133..80c9661e 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -190,7 +190,7 @@ void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &fe, bool with_coef, const IntegrationRule *ir) { - MFEM_VERIFY(ir == NULL, "ir not supported"); + MFEM_VERIFY(ir == nullptr, "ir not supported"); MFEM_VERIFY(NE == pmesh->GetNE(), ""); // ZZ comes with with_coef set to true, not Kelly switch (flux_mode) @@ -218,7 +218,7 @@ void AMREstimatorIntegrator::ComputeElementFlux(const FiniteElement &fe, AMR::AMR(ParMesh *pmesh, int estimator, double ref_t, double jac_t, double deref_t, int max_level, int nc_limit, - double size_b, double energy_b, double *xyz_b): + double size_b, double energy_b, double *blast_xyz): pmesh(pmesh), myid(pmesh->GetMyRank()), dim(pmesh->Dimension()), @@ -228,7 +228,7 @@ AMR::AMR(ParMesh *pmesh, int estimator, opt( { estimator, ref_t, jac_t, deref_t, max_level, nc_limit, - size_b, energy_b, Vertex(xyz_b[0], xyz_b[1], xyz_b[2]) + size_b, energy_b, Vertex(blast_xyz[0], blast_xyz[1], blast_xyz[2]) }) { } AMR::~AMR() { } @@ -244,8 +244,7 @@ void AMR::Setup(ParGridFunction &x_gf) if (opt.estimator == amr::estimator::zz) { - integ = new AMREstimatorIntegrator(pmesh, opt.max_level, - opt.jac_threshold); + integ = new AMREstimatorIntegrator(pmesh, opt.max_level, opt.jac_threshold); smooth_flux_fec = new RT_FECollection(order-1, dim); auto smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec); estimator = new L2ZienkiewiczZhuEstimator(*integ, x_gf, &flux_fes, @@ -296,8 +295,8 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, Array &ess_vdofs) { Vector v_max, v_min; - Array refs; - bool mesh_refined = false; + Array refined; + bool mesh_updated = false; const int NE = pmesh->GetNE(); ParFiniteElementSpace &H1FESpace = *x.ParFESpace(); constexpr double NL_DMAX = std::numeric_limits::max(); @@ -315,7 +314,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, pmesh->pncmesh->GetElementDepth(el) < opt.max_level && (v_min(el) < 1e-3)) // only refine the still area { - refs.Append(Refinement(el)); + refined.Append(Refinement(el)); } } break; @@ -353,7 +352,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, MFEM_VERIFY(rho <= 1.0, ""); if (rho < opt.jac_threshold && depth < opt.max_level) { - refs.Append(Refinement(el)); + refined.Append(Refinement(el)); } } } @@ -364,28 +363,27 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::kelly: { refiner->Apply(*pmesh); - if (refiner->Refined()) { mesh_refined = true; } - MFEM_VERIFY(!refiner->Derefined(),""); - MFEM_VERIFY(!refiner->Rebalanced(),""); + if (refiner->Refined()) { mesh_updated = true; } + MFEM_VERIFY(!refiner->Derefined() && !refiner->Rebalanced(), ""); break; } default: MFEM_ABORT("Unknown AMR estimator!"); } - // custom and JJt uses refs, ZZ and Kelly will set mesh_refined - const int nref = pmesh->ReduceInt(refs.Size()); - if (nref && !mesh_refined) + // custom and JJt use 'refined', ZZ and Kelly will set 'mesh_updated' + const int nref = pmesh->ReduceInt(refined.Size()); + if (nref && !mesh_updated) { constexpr int non_conforming = 1; - pmesh->GeneralRefinement(refs, non_conforming, opt.nc_limit); - mesh_refined = true; + pmesh->GeneralRefinement(refined, non_conforming, opt.nc_limit); + mesh_updated = true; if (myid == 0) { std::cout << "Refined " << nref << " elements." << std::endl; } } else if (opt.estimator == amr::estimator::custom && - opt.deref_threshold >= 0.0 && !mesh_refined) + opt.deref_threshold >= 0.0 && !mesh_updated) { ParGridFunction rho_gf; Vector rho_max, rho_min; @@ -400,8 +398,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, // make sure the blast point is never derefined Array elements; - FindElementsWithVertex(pmesh, opt.blast_position, - opt.blast_size, elements); + FindElementsWithVertex(pmesh, opt.blast_position, opt.blast_size, elements); for (int i = 0; i < elements.Size(); i++) { int index = elements[i]; @@ -414,27 +411,26 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } } - const int op = 2; // maximum value of fine elements - mesh_refined = pmesh->DerefineByError(rho_max, threshold, - opt.nc_limit, op); - if (mesh_refined && myid == 0) + const int op = 2; // operatoe on the 'maximum' value of the fine elements + mesh_updated = pmesh->DerefineByError(rho_max, threshold, opt.nc_limit, op); + if (mesh_updated && myid == 0) { std::cout << "Derefined, threshold = " << threshold << std::endl; } } else if ((opt.estimator == amr::estimator::zz || - opt.estimator == amr::estimator::kelly) && !mesh_refined) + opt.estimator == amr::estimator::kelly) && !mesh_updated) { MFEM_VERIFY(derefiner,""); if (derefiner->Apply(*pmesh)) { //if (myid == 0) { std::cout << "\nDerefined elements." << std::endl; } } - if (derefiner->Derefined()) { mesh_refined = true; } + if (derefiner->Derefined()) { mesh_updated = true; } } else { /* nothing to do */ } - if (mesh_refined) + if (mesh_updated) { auto Update = [&]() { @@ -446,21 +442,21 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, L2FESpace->Update(); MEFESpace->Update(); - const int Vsize_h1 = H1FESpace->GetVSize(); - const int Vsize_l2 = L2FESpace->GetVSize(); + const int h1_vsize = H1FESpace->GetVSize(); + const int l2_vsize = L2FESpace->GetVSize(); true_offset[0] = 0; - true_offset[1] = true_offset[0] + Vsize_h1; - true_offset[2] = true_offset[1] + Vsize_h1; - true_offset[3] = true_offset[2] + Vsize_l2; + true_offset[1] = true_offset[0] + h1_vsize; + true_offset[2] = true_offset[1] + h1_vsize; + true_offset[3] = true_offset[2] + l2_vsize; S_old = S; S.Update(true_offset); - const Operator* H1Update = H1FESpace->GetUpdateOperator(); - const Operator* L2Update = L2FESpace->GetUpdateOperator(); - H1Update->Mult(S_old.GetBlock(0), S.GetBlock(0)); - H1Update->Mult(S_old.GetBlock(1), S.GetBlock(1)); - L2Update->Mult(S_old.GetBlock(2), S.GetBlock(2)); + const Operator* h1_update = H1FESpace->GetUpdateOperator(); + const Operator* l2_update = L2FESpace->GetUpdateOperator(); + h1_update->Mult(S_old.GetBlock(0), S.GetBlock(0)); + h1_update->Mult(S_old.GetBlock(1), S.GetBlock(1)); + l2_update->Mult(S_old.GetBlock(2), S.GetBlock(2)); x.MakeRef(H1FESpace, S, true_offset[0]); v.MakeRef(H1FESpace, S, true_offset[1]); diff --git a/laghos_amr.hpp b/laghos_amr.hpp index a3ec34af..3ced430e 100644 --- a/laghos_amr.hpp +++ b/laghos_amr.hpp @@ -26,7 +26,14 @@ namespace mfem namespace amr { -enum estimator: int { custom = 0, jjt = 1, zz = 2, kelly = 3 }; +enum estimator: int +{ + custom = 0, // adapted from the amr subdir: uses visc and vgrad per element + jjt = 1, // uses rho = minW/maxW, with W is the determinant of the + // adjugate of the transformation's Jacobian. + zz = 2, + kelly = 3 +}; class AMREstimatorIntegrator: public DiffusionIntegrator { diff --git a/laghos_solver.cpp b/laghos_solver.cpp index c689a0c4..a7ef6edd 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -338,8 +338,8 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) // go back to initial mesh configuration temporarily int own_nodes = 0; - GridFunction *x_gf = &x0_gf; - pmesh->SwapNodes(x_gf, own_nodes); + GridFunction *x0 = &x0_gf; + pmesh->SwapNodes(x0, own_nodes); if (p_assembly) { @@ -375,8 +375,8 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) // Setup the preconditioner of the velocity mass operator. // BC are handled by the VMassPA, so ess_tdofs here can be empty. - Array ess_tdofs; - VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), ess_tdofs); + Array empty; + VMassPA_Jprec = new OperatorJacobiSmoother(VMassPA->GetBF(), empty); CG_VMass.SetPreconditioner(*VMassPA_Jprec); CG_VMass.SetOperator(*VMassPA); @@ -455,7 +455,7 @@ void LagrangianHydroOperator::AMRUpdate(const Vector &S, const bool quick) qdata.h0 /= (double) H1.GetOrder(0); // swap back to deformed mesh configuration - pmesh->SwapNodes(x_gf, own_nodes); + pmesh->SwapNodes(x0, own_nodes); } void LagrangianHydroOperator::Mult(const Vector &S, Vector &dS_dt) const @@ -664,7 +664,7 @@ void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) const { rho.SetSpace(&L2); DenseMatrix Mrho(l2dofs_cnt); - Vector rhs(l2dofs_cnt), rho_z(l2dofs_cnt); + Vector rhs_z(l2dofs_cnt), rho_z(l2dofs_cnt); Array dofs(l2dofs_cnt); DenseMatrixInverse inv(&Mrho); MassIntegrator mi(&ir); @@ -674,10 +674,10 @@ void LagrangianHydroOperator::ComputeDensity(ParGridFunction &rho) const { const FiniteElement &fe = *L2.GetFE(e); ElementTransformation &eltr = *L2.GetElementTransformation(e); - di.AssembleRHSElementVect(fe, eltr, rhs); + di.AssembleRHSElementVect(fe, eltr, rhs_z); mi.AssembleElementMatrix(fe, eltr, Mrho); inv.Factor(); - inv.Mult(rhs, rho_z); + inv.Mult(rhs_z, rho_z); L2.GetElementDofs(e, dofs); rho.SetSubVector(dofs, rho_z); } From f8d1c6400eef85859a0db7238117397e541e50c7 Mon Sep 17 00:00:00 2001 From: camierjs Date: Mon, 5 Dec 2022 12:14:24 -0800 Subject: [PATCH 22/22] AMR problems update and glvis keys --- laghos.cpp | 56 +++++++++++++++++++++++++++-------------------- laghos_amr.cpp | 35 +++++++++++++++-------------- laghos_solver.cpp | 4 +++- laghos_solver.hpp | 3 ++- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/laghos.cpp b/laghos.cpp index 3c3aaab6..75e39331 100644 --- a/laghos.cpp +++ b/laghos.cpp @@ -105,6 +105,7 @@ int main(int argc, char *argv[]) bool visualization = false; int vis_steps = 5; int vis_windows = 3; + const char *vis_keys = nullptr; bool visit = false; bool gfprint = false; const char *basename = "results/Laghos"; @@ -170,6 +171,7 @@ int main(int argc, char *argv[]) "Visualize every n-th timestep."); args.AddOption(&vis_windows, "-vw", "--visualization-windows", "Number of visualization windows to open: 1~3"); + args.AddOption(&vis_keys, "-vk", "--visualization-keys", "Keys for vis."); args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit", "Enable or disable VisIt visualization."); args.AddOption(&gfprint, "-print", "--print", "-no-print", "--no-print", @@ -202,11 +204,11 @@ int main(int argc, char *argv[]) args.AddOption(&amr_estimator, "-ae", "--amr-estimator", "AMR estimator: 0:Custom, 1:Rho, 2:ZZ, 3:Kelly"); args.AddOption(&amr_ref_threshold, "-ar", "--amr-ref-threshold", - "AMR Jacobian refinement threshold."); - args.AddOption(&amr_deref_threshold, "-ad", "--amr-deref-threshold", "AMR refinement threshold."); - args.AddOption(&amr_jac_threshold, "-aj", "--amr-jac-threshold", + args.AddOption(&amr_deref_threshold, "-ad", "--amr-deref-threshold", "AMR derefinement threshold (0 = no derefinement)."); + args.AddOption(&amr_jac_threshold, "-aj", "--amr-jac-threshold", + "AMR refinement threshold."); args.AddOption(&amr_max_level, "-am", "--amr-max-level", "AMR max refined level (default to 'rs_levels + rp_levels')"); args.Parse(); @@ -216,17 +218,16 @@ int main(int argc, char *argv[]) return 1; } - amr_max_level = std::max(amr_max_level, rs_levels + rp_levels); - if (mpi.Root()) { args.PrintOptions(cout); } // Check AMR configuration: only Sedov problem (#1) is supported for now - if (amr && problem != 1) + // does this still holds? + /*if (amr && problem != 1) { if (mpi.Root()) { std::cout << "AMR only supported for problem 1." << std::endl; } return 0; - } + }*/ // Configure the device from the command line options Device backend; @@ -288,17 +289,20 @@ int main(int argc, char *argv[]) } // Refine the mesh in serial to increase the resolution. - if (!amr) - { - for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } - } - else + for (int lev = 0; lev < rs_levels; lev++) { mesh->UniformRefinement(); } + //amr_max_level = std::max(amr_max_level, rs_levels + rp_levels);s + + if (amr) { mesh->EnsureNCMesh(); - Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; - for (int lev = 0; lev < rs_levels; lev++) + // refine at the blast position for problem 1 + if (problem == 1) { - mesh->RefineAtVertex(blast, amr_blast_eps); + Vertex blast {blast_position[0], blast_position[1], blast_position[2]}; + for (int lev = 0; lev < rs_levels; lev++) + { + mesh->RefineAtVertex(blast, amr_blast_eps); + } } } @@ -628,24 +632,26 @@ int main(int argc, char *argv[]) vis_v.precision(8); vis_e.precision(8); int Wx = 0, Wy = 0; // window position - const int Ww = 350, Wh = 350; // window size + int Ww = vis_windows==1 ? 1024 : 350, + Wh = vis_windows==1 ? 768 : 350; // window size int offx = Ww+10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, - "Density", Wx, Wy, Ww, Wh); + "Density", vis_keys, Wx, Wy, Ww, Wh); } if (vis_windows > 1) { Wx += offx; hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, - "Velocity", Wx, Wy, Ww, Wh); + "Velocity", vis_keys, Wx, Wy, Ww, Wh); } if (vis_windows > 2) { Wx += offx; hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, - "Specific Internal Energy", Wx, Wy, Ww, Wh); + "Specific Internal Energy", + vis_keys, Wx, Wy, Ww, Wh); } } @@ -787,25 +793,27 @@ int main(int argc, char *argv[]) if (visualization) { int Wx = 0, Wy = 0; // window position - int Ww = 350, Wh = 350; // window size + int Ww = vis_windows==1 ? 1024 : 350, + Wh = vis_windows==1 ? 768 : 350; // window size int offx = Ww+10; // window offsets if (problem != 0 && problem != 4) { hydrodynamics::VisualizeField(vis_rho, vishost, visport, rho_gf, - "Density", Wx, Wy, Ww, Wh); + "Density", vis_keys, Wx, Wy, Ww, Wh); } if (vis_windows > 1) { Wx += offx; - hydrodynamics::VisualizeField(vis_v, vishost, visport, - v_gf, "Velocity", Wx, Wy, Ww, Wh); + hydrodynamics::VisualizeField(vis_v, vishost, visport, v_gf, + "Velocity", + vis_keys, Wx, Wy, Ww, Wh); } if (vis_windows > 2) { Wx += offx; hydrodynamics::VisualizeField(vis_e, vishost, visport, e_gf, "Specific Internal Energy", - Wx, Wy, Ww,Wh); + vis_keys, Wx, Wy, Ww,Wh); } } diff --git a/laghos_amr.cpp b/laghos_amr.cpp index 80c9661e..330917b4 100644 --- a/laghos_amr.cpp +++ b/laghos_amr.cpp @@ -322,7 +322,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, case amr::estimator::jjt: { - const double art = opt.ref_threshold; + const double art = opt.jac_threshold; const int h1_order = H1FESpace.GetOrder(0) + 1; DenseMatrix Jadjt, Jadj(dim, pmesh->SpaceDimension()); MFEM_VERIFY(art >= 0.0, "AMR threshold should be positive"); @@ -350,7 +350,7 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, { const double rho = minW / maxW; MFEM_VERIFY(rho <= 1.0, ""); - if (rho < opt.jac_threshold && depth < opt.max_level) + if (rho < art && depth < opt.max_level) { refined.Append(Refinement(el)); } @@ -382,7 +382,8 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, std::cout << "Refined " << nref << " elements." << std::endl; } } - else if (opt.estimator == amr::estimator::custom && + else if ((opt.estimator == amr::estimator::custom || + opt.estimator == amr::estimator::jjt) && opt.deref_threshold >= 0.0 && !mesh_updated) { ParGridFunction rho_gf; @@ -397,25 +398,27 @@ void AMR::Update(hydrodynamics::LagrangianHydroOperator &hydro, pmesh->GetComm()); // make sure the blast point is never derefined - Array elements; - FindElementsWithVertex(pmesh, opt.blast_position, opt.blast_size, elements); - for (int i = 0; i < elements.Size(); i++) + if (opt.estimator == amr::estimator::custom) { - int index = elements[i]; - if (index >= 0) { rho_max(index) = NL_DMAX; } - } - - // only derefine where the mesh is in motion, i.e. after the shock - for (int i = 0; i < pmesh->GetNE(); i++) - { - if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } + Array elements; + FindElementsWithVertex(pmesh, opt.blast_position, opt.blast_size, elements); + for (int i = 0; i < elements.Size(); i++) + { + int index = elements[i]; + if (index >= 0) { rho_max(index) = NL_DMAX; } + } + // only derefine where the mesh is in motion, i.e. after the shock + for (int i = 0; i < pmesh->GetNE(); i++) + { + if (v_min(i) < 0.1) { rho_max(i) = NL_DMAX; } + } } - const int op = 2; // operatoe on the 'maximum' value of the fine elements + const int op = 2; // operator on the 'maximum' value of the fine elements mesh_updated = pmesh->DerefineByError(rho_max, threshold, opt.nc_limit, op); if (mesh_updated && myid == 0) { - std::cout << "Derefined, threshold = " << threshold << std::endl; + //std::cout << "Derefined, threshold = " << threshold << std::endl; } } else if ((opt.estimator == amr::estimator::zz || diff --git a/laghos_solver.cpp b/laghos_solver.cpp index a7ef6edd..3fa6e8b6 100644 --- a/laghos_solver.cpp +++ b/laghos_solver.cpp @@ -30,6 +30,7 @@ namespace hydrodynamics void VisualizeField(socketstream &sock, const char *vishost, int visport, ParGridFunction &gf, const char *title, + const char *vis_keys, int x, int y, int w, int h, bool vec) { gf.HostRead(); @@ -61,7 +62,8 @@ void VisualizeField(socketstream &sock, const char *vishost, int visport, if (myid == 0 && newly_opened) { - const char* keys = (gf.FESpace()->GetMesh()->Dimension() == 2) + const char* keys = vis_keys ? vis_keys : + (gf.FESpace()->GetMesh()->Dimension() == 2) ? "mAcRjl" : "mmaaAcl"; sock << "window_title '" << title << "'\n" diff --git a/laghos_solver.hpp b/laghos_solver.hpp index fc46b918..66cd7e23 100644 --- a/laghos_solver.hpp +++ b/laghos_solver.hpp @@ -240,6 +240,7 @@ namespace hydrodynamics /// its geometry. void VisualizeField(socketstream &sock, const char *vishost, int visport, ParGridFunction &gf, const char *title, + const char *keys = nullptr, int x = 0, int y = 0, int w = 400, int h = 400, bool vec = false); @@ -355,7 +356,7 @@ class LagrangianHydroOperator : public TimeDependentOperator const int source, const double cfl, const bool visc, const bool vort, const bool pa, - const bool amr, + const bool amr, const double cgt, const int cgiter, double ftz_tol, const int order_q); ~LagrangianHydroOperator();