From 59becf86baf7a00d85b65c48ec1475e85b572ad7 Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Thu, 19 Dec 2024 12:49:03 +0000 Subject: [PATCH 1/2] Fix broken transient problems in parallel by switching to use of ParGridFunction::MakeTRef in ProblemOperators --- .../problem_operators/problem_operator_interface.h | 2 +- .../src/mfem/problem_operators/problem_operator.C | 2 +- .../problem_operators/problem_operator_interface.C | 12 ++---------- .../time_domain_equation_system_problem_operator.C | 4 ++-- .../problem_operators/time_domain_problem_operator.C | 2 +- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/framework/include/mfem/problem_operators/problem_operator_interface.h b/framework/include/mfem/problem_operators/problem_operator_interface.h index 232e0459..ec8e5313 100644 --- a/framework/include/mfem/problem_operators/problem_operator_interface.h +++ b/framework/include/mfem/problem_operators/problem_operator_interface.h @@ -13,7 +13,7 @@ class ProblemOperatorInterface virtual void SetGridFunctions(); virtual void Init(mfem::BlockVector & X); - mfem::Array _true_offsets, _block_true_offsets; + mfem::Array _block_true_offsets; mfem::BlockVector _true_x, _true_rhs; mfem::OperatorHandle _equation_system_operator; diff --git a/framework/src/mfem/problem_operators/problem_operator.C b/framework/src/mfem/problem_operators/problem_operator.C index e6f3c034..45642a2e 100644 --- a/framework/src/mfem/problem_operators/problem_operator.C +++ b/framework/src/mfem/problem_operators/problem_operator.C @@ -7,7 +7,7 @@ void ProblemOperator::SetGridFunctions() { ProblemOperatorInterface::SetGridFunctions(); - width = height = _true_offsets[_trial_variables.size()]; + width = height = _block_true_offsets[_trial_variables.size()]; }; } // namespace platypus \ No newline at end of file diff --git a/framework/src/mfem/problem_operators/problem_operator_interface.C b/framework/src/mfem/problem_operators/problem_operator_interface.C index 9b20d658..dcc23924 100644 --- a/framework/src/mfem/problem_operators/problem_operator_interface.C +++ b/framework/src/mfem/problem_operators/problem_operator_interface.C @@ -17,14 +17,6 @@ ProblemOperatorInterface::SetGridFunctions() } _block_true_offsets.PartialSum(); - _true_offsets.SetSize(_trial_variables.size() + 1); - _true_offsets[0] = 0; - for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind) - { - _true_offsets[ind + 1] = _trial_variables.at(ind)->ParFESpace()->GetVSize(); - } - _true_offsets.PartialSum(); - _true_x.Update(_block_true_offsets); _true_rhs.Update(_block_true_offsets); } @@ -32,10 +24,10 @@ ProblemOperatorInterface::SetGridFunctions() void ProblemOperatorInterface::Init(mfem::BlockVector & X) { - X.Update(_true_offsets); + X.Update(_block_true_offsets); for (size_t i = 0; i < _test_variables.size(); ++i) { - _test_variables.at(i)->MakeRef(_test_variables.at(i)->ParFESpace(), X, _true_offsets[i]); + _test_variables.at(i)->MakeTRef(_test_variables.at(i)->ParFESpace(), X, _block_true_offsets[i]); } } diff --git a/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C b/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C index 3525df68..09e290ef 100644 --- a/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C +++ b/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C @@ -29,8 +29,8 @@ TimeDomainEquationSystemProblemOperator::ImplicitSolve(const double dt, dX_dt = 0.0; for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind) { - _trial_variables.at(ind)->MakeRef( - _trial_variables.at(ind)->ParFESpace(), dX_dt, _true_offsets[ind]); + _trial_variables.at(ind)->MakeTRef( + _trial_variables.at(ind)->ParFESpace(), dX_dt, _block_true_offsets[ind]); } const double time = GetTime(); for (auto & coef : _problem._scalar_manager) diff --git a/framework/src/mfem/problem_operators/time_domain_problem_operator.C b/framework/src/mfem/problem_operators/time_domain_problem_operator.C index 7b8f7b6c..9f61465b 100644 --- a/framework/src/mfem/problem_operators/time_domain_problem_operator.C +++ b/framework/src/mfem/problem_operators/time_domain_problem_operator.C @@ -17,7 +17,7 @@ void TimeDomainProblemOperator::SetGridFunctions() { ProblemOperatorInterface::SetGridFunctions(); - width = height = _true_offsets[_trial_variables.size()]; + width = height = _block_true_offsets[_trial_variables.size()]; } } // namespace platypus \ No newline at end of file From 087d573123372f56a1a71c138ef6ba38e1d591ee Mon Sep 17 00:00:00 2001 From: Alexander Blair Date: Thu, 19 Dec 2024 16:46:59 +0000 Subject: [PATCH 2/2] Fix issue causing parallel transient problems to output a zero solution --- .../problem_operator_interface.h | 2 ++ .../src/mfem/executioners/MFEMTransient.C | 3 +++ .../problem_operator_interface.C | 18 ++++++++++++++++++ ...e_domain_equation_system_problem_operator.C | 2 ++ 4 files changed, 25 insertions(+) diff --git a/framework/include/mfem/problem_operators/problem_operator_interface.h b/framework/include/mfem/problem_operators/problem_operator_interface.h index ec8e5313..38e59590 100644 --- a/framework/include/mfem/problem_operators/problem_operator_interface.h +++ b/framework/include/mfem/problem_operators/problem_operator_interface.h @@ -11,6 +11,8 @@ class ProblemOperatorInterface virtual ~ProblemOperatorInterface() = default; virtual void SetGridFunctions(); + virtual void SetTestVariablesFromTrueVectors(); + virtual void SetTrialVariablesFromTrueVectors(); virtual void Init(mfem::BlockVector & X); mfem::Array _block_true_offsets; diff --git a/framework/src/mfem/executioners/MFEMTransient.C b/framework/src/mfem/executioners/MFEMTransient.C index 4f67d7bf..f113939e 100644 --- a/framework/src/mfem/executioners/MFEMTransient.C +++ b/framework/src/mfem/executioners/MFEMTransient.C @@ -52,6 +52,9 @@ MFEMTransient::step(double dt, int it) const // Advance time step. _problem_data._ode_solver->Step(_problem_data._f, _t, dt); + // Synchonise time dependent GridFunctions with updated DoF data. + _problem_operator->SetTestVariablesFromTrueVectors(); + // Sync Host/Device _problem_data._f.HostRead(); diff --git a/framework/src/mfem/problem_operators/problem_operator_interface.C b/framework/src/mfem/problem_operators/problem_operator_interface.C index dcc23924..637b1017 100644 --- a/framework/src/mfem/problem_operators/problem_operator_interface.C +++ b/framework/src/mfem/problem_operators/problem_operator_interface.C @@ -31,4 +31,22 @@ ProblemOperatorInterface::Init(mfem::BlockVector & X) } } +void +ProblemOperatorInterface::SetTestVariablesFromTrueVectors() +{ + for (unsigned int ind = 0; ind < _test_variables.size(); ++ind) + { + _test_variables.at(ind)->SetFromTrueVector(); + } +} + +void +ProblemOperatorInterface::SetTrialVariablesFromTrueVectors() +{ + for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind) + { + _trial_variables.at(ind)->SetFromTrueVector(); + } +} + } \ No newline at end of file diff --git a/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C b/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C index 09e290ef..17cdbb58 100644 --- a/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C +++ b/framework/src/mfem/problem_operators/time_domain_equation_system_problem_operator.C @@ -27,6 +27,7 @@ TimeDomainEquationSystemProblemOperator::ImplicitSolve(const double dt, mfem::Vector & dX_dt) { dX_dt = 0.0; + SetTestVariablesFromTrueVectors(); for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind) { _trial_variables.at(ind)->MakeTRef( @@ -50,6 +51,7 @@ TimeDomainEquationSystemProblemOperator::ImplicitSolve(const double dt, _problem._nonlinear_solver->SetSolver(*_problem._jacobian_solver); _problem._nonlinear_solver->SetOperator(*GetEquationSystem()); _problem._nonlinear_solver->Mult(_true_rhs, dX_dt); + SetTrialVariablesFromTrueVectors(); } void