From 0babdbd92bb74e1f374dc2f020a01ac248cac96c Mon Sep 17 00:00:00 2001
From: Alexander Blair <alexander.blair@ukaea.uk>
Date: Thu, 19 Dec 2024 12:49:03 +0000
Subject: [PATCH] Fix broken transient problems in parallel by switching to use
 of ParGridFunction::MakeTRef in ProblemOperators

---
 .../problem_operators/problem_operator_interface.h   |  2 +-
 src/problem_operators/problem_operator.C             |  2 +-
 src/problem_operators/problem_operator_interface.C   | 12 ++----------
 .../time_domain_equation_system_problem_operator.C   |  4 ++--
 src/problem_operators/time_domain_problem_operator.C |  2 +-
 5 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/include/problem_operators/problem_operator_interface.h b/include/problem_operators/problem_operator_interface.h
index 232e0459..ec8e5313 100644
--- a/include/problem_operators/problem_operator_interface.h
+++ b/include/problem_operators/problem_operator_interface.h
@@ -13,7 +13,7 @@ class ProblemOperatorInterface
   virtual void SetGridFunctions();
   virtual void Init(mfem::BlockVector & X);
 
-  mfem::Array<int> _true_offsets, _block_true_offsets;
+  mfem::Array<int> _block_true_offsets;
 
   mfem::BlockVector _true_x, _true_rhs;
   mfem::OperatorHandle _equation_system_operator;
diff --git a/src/problem_operators/problem_operator.C b/src/problem_operators/problem_operator.C
index e6f3c034..45642a2e 100644
--- a/src/problem_operators/problem_operator.C
+++ b/src/problem_operators/problem_operator.C
@@ -7,7 +7,7 @@ void
 ProblemOperator::SetGridFunctions()
 {
   ProblemOperatorInterface::SetGridFunctions();
-  width = height = _true_offsets[_trial_variables.size()];
+  width = height = _block_true_offsets[_trial_variables.size()];
 };
 
 } // namespace platypus
\ No newline at end of file
diff --git a/src/problem_operators/problem_operator_interface.C b/src/problem_operators/problem_operator_interface.C
index 9b20d658..dcc23924 100644
--- a/src/problem_operators/problem_operator_interface.C
+++ b/src/problem_operators/problem_operator_interface.C
@@ -17,14 +17,6 @@ ProblemOperatorInterface::SetGridFunctions()
   }
   _block_true_offsets.PartialSum();
 
-  _true_offsets.SetSize(_trial_variables.size() + 1);
-  _true_offsets[0] = 0;
-  for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind)
-  {
-    _true_offsets[ind + 1] = _trial_variables.at(ind)->ParFESpace()->GetVSize();
-  }
-  _true_offsets.PartialSum();
-
   _true_x.Update(_block_true_offsets);
   _true_rhs.Update(_block_true_offsets);
 }
@@ -32,10 +24,10 @@ ProblemOperatorInterface::SetGridFunctions()
 void
 ProblemOperatorInterface::Init(mfem::BlockVector & X)
 {
-  X.Update(_true_offsets);
+  X.Update(_block_true_offsets);
   for (size_t i = 0; i < _test_variables.size(); ++i)
   {
-    _test_variables.at(i)->MakeRef(_test_variables.at(i)->ParFESpace(), X, _true_offsets[i]);
+    _test_variables.at(i)->MakeTRef(_test_variables.at(i)->ParFESpace(), X, _block_true_offsets[i]);
   }
 }
 
diff --git a/src/problem_operators/time_domain_equation_system_problem_operator.C b/src/problem_operators/time_domain_equation_system_problem_operator.C
index 3525df68..09e290ef 100644
--- a/src/problem_operators/time_domain_equation_system_problem_operator.C
+++ b/src/problem_operators/time_domain_equation_system_problem_operator.C
@@ -29,8 +29,8 @@ TimeDomainEquationSystemProblemOperator::ImplicitSolve(const double dt,
   dX_dt = 0.0;
   for (unsigned int ind = 0; ind < _trial_variables.size(); ++ind)
   {
-    _trial_variables.at(ind)->MakeRef(
-        _trial_variables.at(ind)->ParFESpace(), dX_dt, _true_offsets[ind]);
+    _trial_variables.at(ind)->MakeTRef(
+        _trial_variables.at(ind)->ParFESpace(), dX_dt, _block_true_offsets[ind]);
   }
   const double time = GetTime();
   for (auto & coef : _problem._scalar_manager)
diff --git a/src/problem_operators/time_domain_problem_operator.C b/src/problem_operators/time_domain_problem_operator.C
index 7b8f7b6c..9f61465b 100644
--- a/src/problem_operators/time_domain_problem_operator.C
+++ b/src/problem_operators/time_domain_problem_operator.C
@@ -17,7 +17,7 @@ void
 TimeDomainProblemOperator::SetGridFunctions()
 {
   ProblemOperatorInterface::SetGridFunctions();
-  width = height = _true_offsets[_trial_variables.size()];
+  width = height = _block_true_offsets[_trial_variables.size()];
 }
 
 } // namespace platypus
\ No newline at end of file