properly report different jumpprocesses in math description

.github/workflows/NightlyBMDB_CLI.yml

@@ -5,12 +5,12 @@ on:
   workflow_dispatch:
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
   poetry-version: "1.2.1"
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Our Code
         uses: actions/checkout@v4

.github/workflows/ci_cd.yml

@@ -10,7 +10,7 @@ on:
     types: [published, edited]
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
 
 jobs:
   CI-Build:

.github/workflows/create_bs_singularity.yml

@@ -8,12 +8,12 @@ on:
         default: 3ca1093da599a0ff4229b1130efe7a3924ff2b4c
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
 
 jobs:
   CD:
     name: CD
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/nightly-model-testing.yml

@@ -6,12 +6,12 @@ on:
   workflow_dispatch:
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
   poetry-version: "1.2.1"
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Our Code
         uses: actions/checkout@v4

docker/build/Dockerfile-batch-dev

@@ -1,4 +1,4 @@
-FROM ghcr.io/virtualcell/vcell-solvers:v0.8.1.3
+FROM ghcr.io/virtualcell/vcell-solvers:v0.8.2
 
 RUN apt-get -y update && \
     apt-get install -y  curl && \

docker/build/Dockerfile-data-dev

@@ -10,7 +10,7 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM python:3.9.7-slim
+FROM python:3.10.15-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME

vcell-cli-utils/vcell_cli_utils/cli.py

@@ -28,7 +28,6 @@
 from biosimulators_utils.report.io import ReportWriter
 from biosimulators_utils.sedml.data_model import Report, Plot2D, Plot3D, DataSet
 from biosimulators_utils.sedml.io import SedmlSimulationReader, SedmlSimulationWriter
-from deprecated import deprecated
 from libsedml import SedReport, SedPlot2D, SedDocument
 
 # Move status PY code here
@@ -140,97 +139,6 @@ def gen_sedml_2d_3d(omex_file_path, base_out_path):
     return temp_path
 
 
-@deprecated("This method is no longer used")
-def exec_sed_doc(omex_file_path, base_out_path):
-    # defining archive
-    config = Config(VALIDATE_OMEX_MANIFESTS=False)
-    archive = CombineArchiveReader().run(in_file=omex_file_path, out_dir=tmp_dir,
-                                         config=config)
-
-    # determine files to execute
-    sedml_contents = get_sedml_contents(archive)
-
-    report_results = ReportResults()
-    for i_content, content in enumerate(sedml_contents):
-        content_filename = os.path.join(tmp_dir, content.location)
-
-        doc = SedmlSimulationReader().run(content_filename)
-
-        for report_filename in glob.glob(os.path.join(base_out_path, content.location, '*.csv')):
-            report_id = os.path.splitext(os.path.basename(report_filename))[0]
-
-            # read report from CSV file produced by VCell
-            data_set_df = pd.read_csv(report_filename).transpose()
-            data_set_df.columns = data_set_df.iloc[0]
-            data_set_df = data_set_df.drop(data_set_df.iloc[0].name)
-            data_set_df = data_set_df.reset_index()
-            data_set_df = data_set_df.rename(
-                columns={'index': data_set_df.columns.name})
-            data_set_df = data_set_df.transpose()
-            data_set_df.index.name = None
-
-            report = next(
-                report for report in doc.outputs if report.id == report_id)
-
-            data_set_results = DataSetResults()
-
-            # print("report: ", report, file=sys.stderr)
-            # print("report Type: ", type(report), file=sys.stderr)
-            # print("Plot Type: ", Plot2D, file=sys.stderr)
-            if type(report) != Plot2D and type(report) != Plot3D:
-                # Considering the scenario where it has the datasets in sedml
-                for data_set in report.data_sets:
-                    data_set_results[data_set.id] = data_set_df.loc[data_set.label, :].to_numpy(
-                        dtype='float64')
-                    # print("DF for report: ", data_set_results[data_set.id], file=sys.stderr)
-                    # print("df.types: ", data_set_results[data_set.id].dtype, file=sys.stderr)
-            else:
-                data_set_df = pd.read_csv(report_filename, header=None).T
-                data_set_df.columns = data_set_df.iloc[0]
-                data_set_df.drop(0, inplace=True)
-                data_set_df.reset_index(inplace=True)
-                data_set_df.drop('index', axis=1, inplace=True)
-                # print("DF for plot: ", data_set_df, file=sys.stderr)
-                # Considering the scenario where it doesn't have datasets in sedml (pseudo sedml for plots)
-                for col in list(data_set_df.columns):
-                    data_set_results[col] = data_set_df[col].values
-
-            # append to data structure of report results
-            report_results[report_id] = data_set_results
-
-            # save file in desired BioSimulators format(s)
-            # for report_format in report_formats:
-            # print("HDF report: ", report, file=sys.stderr)
-            # print("HDF dataset results: ", data_set_results, file=sys.stderr)
-            # print("HDF base_out_path: ", base_out_path,file=sys.stderr)
-            # print("HDF path: ", os.path.join(content.location, report.id), file=sys.stderr)
-
-            rel_path = os.path.join(content.location, report.id)
-
-            if len(rel_path.split("./")) > 1:
-                rel_path = rel_path.split("./")[1]
-
-            if type(report) != Plot2D and type(report) != Plot3D:
-                ReportWriter().run(report,
-                                   data_set_results,
-                                   base_out_path,
-                                   rel_path,
-                                   format='h5')
-            else:
-                datasets = []
-                for col in list(data_set_df.columns):
-                    datasets.append(DataSet(id=col, label=col, name=col))
-                report.data_sets = datasets
-                ReportWriter().run(report,
-                                   data_set_results,
-                                   base_out_path,
-                                   rel_path,
-                                   format='h5')
-
-    # Remove temp directory
-    shutil.rmtree(tmp_dir)
-
-
 def transpose_vcml_csv(csv_file_path: str):
     df = pd.read_csv(csv_file_path, header=None)
     cols = list(df.columns)
@@ -451,7 +359,6 @@ def gen_plots_for_sed2d_only(sedml_path, result_out_dir):
     fire.Fire({
         'genSedml2d3d': gen_sedml_2d_3d,
         'genPlotsPseudoSedml': gen_plots_for_sed2d_only,
-        'execSedDoc': exec_sed_doc,
         'transposeVcmlCsv': transpose_vcml_csv,
         'genPlotPdfs': gen_plot_pdfs,
     })

vcell-cli-utils/vcell_cli_utils/wrapper.py

@@ -2,7 +2,6 @@
 import sys
 import glob
 import os
-from deprecated import deprecated
 from pathlib import Path
 if __name__ == "__main__":
     from vcell_cli_utils import cli, status
@@ -35,7 +34,7 @@ def genSedml2d3d(omexFilePath : str, baseOutPath : str) -> None:
 def genPlotsPseudoSedml(sedmlPath : str, resultOutDir : str) -> None:
     cli.gen_plots_for_sed2d_only(sedmlPath, resultOutDir)
 
-@deprecated("This method is no longer used") 
+# this function is no longer used
 def execSedDoc(omexFilePath : str, baseOutPath : str) -> None:
     cli.exec_sed_doc(omexFilePath, baseOutPath)
 

vcell-cli/src/main/java/org/vcell/cli/run/PythonCalls.java

@@ -72,20 +72,6 @@ public static void setExceptionMessage(String sedmlAbsolutePath, String entityId
         cliPythonManager.parsePythonReturn(results, "", "Failed updating task status YAML\n");
     }
 
-    public void convertCSVtoHDF(String omexFilePath, String outputDir, CLIPythonManager cliPythonManager) throws PythonStreamException {
-
-        // Convert CSV to HDF5
-        /*
-        Usage: cli.py SEDML_FILE_PATH WORKING_DIR BASE_OUT_PATH CSV_DIR <flags>
-                    optional flags:        --rel_out_path | --apply_xml_model_changes |
-                         --report_formats | --plot_formats | --log | --indent
-        * */
-        // handle exceptions here
-        logger.trace("Dialing Python function execSedDoc");
-        String results = cliPythonManager.callPython("execSedDoc", omexFilePath, outputDir);
-        cliPythonManager.parsePythonReturn(results, "HDF conversion successful\n","HDF conversion failed\n");
-    }
-
     // Sample STATUS YML
     /*
     sedDocuments:

vcell-core/src/main/java/cbit/vcell/math/JumpProcess.java

@@ -12,6 +12,7 @@
 import java.util.ArrayList;
 import java.util.Vector;
 
+import cbit.vcell.parser.ExpressionUtils;
 import org.vcell.util.CommentStringTokenizer;
 import org.vcell.util.Compare;
 
@@ -78,7 +79,8 @@ public boolean compareEqual(org.vcell.util.Matchable object)
 
 	JumpProcess jumpProc = (JumpProcess) object;
 	if(processName.compareTo(jumpProc.processName) != 0 ) return false;//processName
-	if(!probabilityRate.compareEqual(jumpProc.probabilityRate)) return false; //probabilityRate
+	var predicate = new ExpressionUtils.ExpressionEquivalencePredicate();
+	if(!Compare.isEqual(probabilityRate, jumpProc.probabilityRate, predicate)) return false;
 	//actions
 	if((actions != null) && (jumpProc.actions != null))
 	{

vcell-core/src/main/java/cbit/vcell/math/MathCompareResults.java

@@ -33,6 +33,8 @@ public enum Decision {
 		MathDifferent_EQUATION_REMOVED(false,"MathDifferent:EquationRemoved"),
 		MathDifferent_UNKNOWN_DIFFERENCE_IN_EQUATION(false,"MathDifferent:UnknownDifferenceInEquation"),
 		MathDifferent_DIFFERENT_PARTICLE_PROPERTIES(false,"MathDifferent:DifferentParticleProperties"),
+		MathDifferent_DIFFERENT_NUMBER_OF_JUMP_PROCESS(false,"MathDifferent:DifferentNumJumpProcs"),
+		MathDifferent_DIFFERENT_JUMP_PROCESS(false,"MathDifferent:DifferentJumpProc"),
 		MathDifferent_DIFFERENT_NUMBER_OF_PARTICLE_JUMP_PROCESS(false,"MathDifferent:DifferentNumParticleJumpProcs"),
 		MathDifferent_DIFFERENT_PARTICLE_JUMP_PROCESS(false,"MathDifferent:DifferentParticleJumpProc"),
 		MathDifferent_LEGACY_RATE_PARTICLE_JUMP_PROCESS(false,"MathDifferent:LegacyRateParticleJumpProc"),

vcell-core/src/main/java/cbit/vcell/math/MathDescription.java

@@ -881,6 +881,46 @@ private MathCompareResults compareEquivalentCanonicalMath(MathDescription newMat
                             return new MathCompareResults(Decision.MathDifferent_DIFFERENT_PARTICLE_JUMP_PROCESS, msg);
                         }
                     }
+
+                    //
+                    // 1) get list of old and new JumpProcesses
+                    // 2) filter out 'no-op' JPs (e.g. those only with trivial Actions such as open->open)
+                    // 3) add trivial Symmetry factor (1.0) if missing
+                    //
+                    List<JumpProcess> oldJpList = subDomainsOld.get(i).getJumpProcesses();
+                    List<JumpProcess> newJpList = subDomainsNew.get(i).getJumpProcesses();
+
+                    if(oldJpList.size() != newJpList.size()){
+                        Set<String> oldJpNameSet = oldJpList.stream().map(JumpProcess::getName).collect(Collectors.toSet());
+                        Set<String> newJpNameSet = newJpList.stream().map(JumpProcess::getName).collect(Collectors.toSet());
+                        Set<String> removedJpNames = new LinkedHashSet<>(oldJpNameSet);
+                        removedJpNames.removeAll(newJpNameSet);
+                        Set<String> addedJpNames = new LinkedHashSet<>(newJpNameSet);
+                        addedJpNames.removeAll(oldJpNameSet);
+                        String msg = "removed JPs=" + removedJpNames + ", added JPs=" + addedJpNames;
+                        logMathTexts(this, newMathDesc, Decision.MathDifferent_DIFFERENT_NUMBER_OF_JUMP_PROCESS, msg);
+                        return new MathCompareResults(Decision.MathDifferent_DIFFERENT_NUMBER_OF_JUMP_PROCESS, msg);
+                    }
+                    for(JumpProcess oldJp : oldJpList){
+                        boolean bEqual = false;
+                        JumpProcess matchingJp = null;
+                        for(JumpProcess newJp : newJpList){
+                            if(Compare.isEqualOrNull(oldJp.getName(), newJp.getName())){
+                                matchingJp = newJp;
+                                if(oldJp.compareEqual(newJp)){
+                                    bEqual = true;
+                                }
+                                break;
+                            }
+                        }
+                        if(!bEqual){
+                            String oldExprList = Arrays.stream(oldJp.getExpressions()).map(Expression::infix).toList().toString();
+                            String newExprList = ((matchingJp == null) ? "null" : Arrays.stream(matchingJp.getExpressions()).map(Expression::infix).toList().toString());
+                            String msg = "JP='" + oldJp.getName() + "', " + "rate: " + "old='" + oldExprList + "', new='" + newExprList + "'";
+                            logMathTexts(this, newMathDesc, Decision.MathDifferent_DIFFERENT_JUMP_PROCESS, msg);
+                            return new MathCompareResults(Decision.MathDifferent_DIFFERENT_JUMP_PROCESS, msg);
+                        }
+                    }
                 }
 
                 //

vcell-core/src/test/java/cbit/vcell/mapping/MathGenCompareTest.java

@@ -130,11 +130,14 @@ public static Set<String> knownMathGenerationFailures() {
 	public static Map<String, MathCompareResults.Decision> knownLegacyFaults() {
 		HashMap<String, MathCompareResults.Decision> faults = new HashMap();
 		faults.put("lumped_reaction_proper_size_in_rate.vcml:Application0", MathCompareResults.Decision.MathDifferent_DIFFERENT_EXPRESSION); // (not saved): expressions are different: ' - (3.321077566325453E-8 * s1)' vs ' - (0.001660538783162726 * s1)'
+		faults.put("biomodel_185577495.vcml:Copy of Application0 1",MathCompareResults.Decision.MathDifferent_DIFFERENT_NUMBER_OF_JUMP_PROCESS); // (97681008:2020-07-30:Private): removed PJPs=[cAMPind_reverse], added PJPs=[]
 		faults.put("biomodel_47429473.vcml:NWASP at Lam Tip in 3D Geometry", MathCompareResults.Decision.MathDifferent_DIFFERENT_FASTINV_EXPRESSION); // (les:6:2010-08-12:Public): could not find a match for fast invariant expression'Expression@b29ced42 '(BarbedD_Cyt - Prof_Cyt + BarbedDPi_Cyt + BarbedT_Cyt)''
 		faults.put("biomodel_55178308.vcml:Spatial 1 - 3D -  electrophysiology", MathCompareResults.Decision.MathDifferent_FAILURE_UNKNOWN); // (Brown:26331356:2011-03-12:Private): line #630 Exception: variable Na not defined
 		faults.put("biomodel_97075423.vcml:NFSim", MathCompareResults.Decision.MathDifferent_LEGACY_SYMMETRY_PARTICLE_JUMP_PROCESS); // (mblinov:12487253:2015-10-07:Private): PJP='r1', ProcessSymmetryFactor: old='1.0', new='0.5'
 		faults.put("biomodel_97705317.vcml:NFSim app", MathCompareResults.Decision.MathDifferent_DIFFERENT_PARTICLE_JUMP_PROCESS); // (BioNetGen:95093638:2015-11-23:Public): PJP='r10', SymmetryFactor: old='0.5' new='0.5', rate: old='[0.0016611295681063123]', new='[0.0033222591362126247]'
 		faults.put("biomodel_97786619.vcml:NFSim app", MathCompareResults.Decision.MathDifferent_DIFFERENT_PARTICLE_JUMP_PROCESS); // (BioNetGen:95093638:2015-12-02:Public): PJP='r10', SymmetryFactor: old='0.5' new='0.5', rate: old='[0.0016611295681063123]', new='[0.0033222591362126247]'
+		faults.put("biomodel_97786886.vcml:ssa",MathCompareResults.Decision.MathDifferent_DIFFERENT_JUMP_PROCESS); // (17:2015-12-02:Public): JP='r1', rate: old='[(1.6611295681061237E-5 * ca_tt_Count * open_Count), -1.0, 0.0, 1.0]', new='[(3.3222591362126247E-6 * ca_tt_Count * open_Count), -1.0, 0.0, 1.0]'
+		faults.put("biomodel_97787114.vcml:ssa",MathCompareResults.Decision.MathDifferent_DIFFERENT_JUMP_PROCESS); // (17:2015-12-02:Public): JP='r1', rate: old='[(1.6611295681061237E-5 * ca_tt_Count * open_Count), -1.0, 0.0, 1.0]', new='[(3.3222591362126247E-6 * ca_tt_Count * open_Count), -1.0, 0.0, 1.0]'
 
 		// all private legacy Mass Action models with no reactant and no-zero Kf (all rest of models identified in issue 536 are fixed)
 		faults.put("biomodel_115999897.vcml:Application0",MathCompareResults.Decision.MathDifferent_DIFFERENT_EXPRESSION); // (60339657:2017-08-06:Private): expressions are different: '(PI45P2 - (0.03 * IP3))' vs '((0.0928262 * PI45P2) - (0.03 * IP3))'

vcell-core/src/test/java/org/vcell/sbml/SEDMLExporterSBMLTest.java

@@ -90,6 +90,7 @@ Map<String, SEDMLExporterCommon.SEDML_FAULT> knownSEDMLFaults() {
 		faults.put("biomodel_169993006.vcml", SEDML_FAULT.NO_MODELS_IN_OMEX);
 		faults.put("biomodel_17098642.vcml", SEDML_FAULT.NO_MODELS_IN_OMEX);
 		faults.put("biomodel_17257105.vcml", SEDML_FAULT.MATH_OVERRIDE_NAMES_DIFFERENT);  // simulation '1 uM iso' in simContext 'compartmental'
+		faults.put("biomodel_185577495.vcml", SEDML_FAULT.MATH_OVERRIDE_NAMES_DIFFERENT); // simulation 'done 1 2 1' in simContext 'Copy of Application0 1'
 		faults.put("biomodel_188880263.vcml", SEDML_FAULT.NO_MODELS_IN_OMEX);
 		faults.put("biomodel_18894555.vcml", SEDML_FAULT.MATH_OVERRIDE_NOT_EQUIVALENT);  // simulation 'baseline' in simContext 'compartmental'
 		faults.put("biomodel_189321805.vcml", SEDML_FAULT.NO_MODELS_IN_OMEX);  // new
@@ -325,15 +326,16 @@ public static Collection<TestCase> testCases() {
 				!largeFileSet().contains(t) &&
 				!slowTestSet().contains(t);
 		Stream<TestCase> sbml_test_cases = Arrays.stream(VcmlTestSuiteFiles.getVcmlTestCases()).filter(skipFilter_SBML).map(fName -> new TestCase(fName, ModelFormat.SBML));
-		return sbml_test_cases.collect(Collectors.toList());
-//		return Arrays.asList(
+//		return sbml_test_cases.collect(Collectors.toList());
+		return Arrays.asList(
+				new TestCase("biomodel_185577495.vcml", ModelFormat.SBML)
 //				new TestCase("biomodel_31523791.vcml", ModelFormat.SBML),
 //				new TestCase("biomodel_34855932.vcml", ModelFormat.SBML),
 //				new TestCase("biomodel_40882931.vcml", ModelFormat.SBML),
 //				new TestCase("biomodel_40883509.vcml", ModelFormat.SBML),
 //				new TestCase("biomodel_65311813.vcml", ModelFormat.SBML),
 //				new TestCase("biomodel_155016832.vcml", ModelFormat.SBML)
-//				);
+				);
 	}
 
 	@ParameterizedTest

vcell-math/src/main/java/cbit/vcell/parser/ExpressionUtils.java

@@ -423,7 +423,7 @@ public static boolean functionallyEquivalent(Expression exp1, Expression exp2, b
 		if (bFirstAnswer == bSecondAnswer){
 			return bFirstAnswer;
 		}else{
-			System.out.println("first  '"+exp1.infix()+"=="+exp2.infix()+"':"+bFirstAnswer+"'\nsecond '"+exp1.flattenSafe().infix()+"=="+exp2.flattenSafe().infix()+"':"+bSecondAnswer+"'");
+			lg.debug("first  '"+exp1.infix()+"=="+exp2.infix()+"':"+bFirstAnswer+"'\nsecond '"+exp1.flattenSafe().infix()+"=="+exp2.flattenSafe().infix()+"':"+bSecondAnswer+"'");
 			return bFirstAnswer;
 		}
 	} catch (ExpressionException e) {