MPI support for spatial ecolab.

Added an MPI-enabled python interpreter

Makefile

@@ -24,7 +24,7 @@ MCFG=include/Makefile.config
 
 ifneq ($(MAKECMDGOALS),clean)
 # make sure Classdesc is built first, even before starting to include Makefiles
-build_classdesc:=$(shell if [ ! -x bin/classdesc ]; then cd classdesc; $(MAKE) XDR=$(XDR); fi)
+build_classdesc:=$(shell if [ ! -x bin/classdesc ]; then pushd classdesc; $(MAKE) XDR=$(XDR); popd; ln -sf `pwd`/classdesc/classdesc bin/classdesc; fi)
 endif
 
 include include/Makefile
@@ -118,7 +118,7 @@ all: all-without-models
 	$(MAKE) models 
 	-$(CHMOD) a+x models/*.tcl
 
-all-without-models: ecolab-libs lib/libecolab$(ECOLIBS_EXT).a
+all-without-models: ecolab-libs lib/libecolab$(ECOLIBS_EXT).a bin/ecolab$(ECOLIBS_EXT)
 	-$(CHMOD) a+x $(SCRIPTS)
 # copy in the system built TCL library
 ifdef MXE
@@ -194,7 +194,7 @@ ifeq ($(OS),Darwin)
 endif
 	$(CPLUSPLUS) -shared -Wl,-soname,libecolab$(ECOLIBS_EXT).so.$(SOVERSION)  $(OBJS) $(LIBMODS) graphcode/*.o $(LIBS) -o lib/libecolab$(ECOLIBS_EXT).so.$(SOVERSION)
 	cd lib; ln -sf libecolab$(ECOLIBS_EXT).so.$(SOVERSION) libecolab$(ECOLIBS_EXT).so
-	cd lib; ln -sf libecolab$(ECOLIBS_EXT).so.$(SOVERSION) ecolab$(ECOLIBS_EXT).so
+	cd lib; ln -sf libecolab$(ECOLIBS_EXT).so.$(SOVERSION) ecolab.so
 
 $(MODS:%=lib/%): lib/%: src/%
 	cp $< $@
@@ -253,10 +253,10 @@ distrib: doc/ecolab/ecolab.html
 latex-docs:
 	if which latex; then cd doc; rm -f *.aux *.dvi *.log *.blg *.toc *.lof *.out; latex -interaction=batchmode ecolab; fi
 
-# bin/ecolab is a no model ecolab binary that only works from installed location
-#bin/ecolab$(ECOLIBS_EXT): src/ecolab.o src/tclmain.o lib/libecolab$(ECOLIBS_EXT).a
-#	$(LINK) $(FLAGS) src/ecolab.o src/tclmain.o  $(LIBS) -o $@
-#	-find . \( -name "*.cc" -o -name "*.h" \) -print |etags -
+#bin/ecolab is a python interpreter supporting MPI
+bin/ecolab$(ECOLIBS_EXT): src/ecolab.o src/pythonMain.o lib/libecolab$(ECOLIBS_EXT).a
+	$(LINK) $(FLAGS) src/ecolab.o src/pythonMain.o -Wl,-rpath $(ECOLAB_HOME)/lib $(LIBS) $(shell pkg-config --libs python3) -o $@
+	-find . \( -name "*.cc" -o -name "*.h" \) -print |etags -
 
 .PHONY: install
 install: all-without-models

lib/GUI.py

@@ -35,4 +35,7 @@ def gui(step):
     ttk.Button(buttonBar,text="run",command=simulator).pack(side='left')
     ttk.Button(buttonBar,text="step",command=step).pack(side='left')
     ttk.Button(buttonBar,text="stop",command=simulator.stop).pack(side='left')
-    runner.mainloop()
+    try:
+        runner.mainloop()
+    except SystemExit:
+        pass

models/ecolab_model.cc

@@ -142,7 +142,13 @@ void SpatialModel::condense()
 {
   array<int> total_density(species.size());
   for (auto& i: *this) total_density+=i->as<EcolabCell>()->density;
+#ifdef MPI_SUPPORT
+  array<int> recv(total_density.size());
+  MPI_Allreduce(total_density.data(),recv.data(),total_density.size(),MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+  total_density.swap(recv);
+#endif
   auto mask=total_density != 0;
+
   size_t mask_true=sum(mask);
   if (mask.size()==mask_true) return; /* no change ! */
   ModelData::condense(mask,mask_true);
@@ -170,27 +176,46 @@ void SpatialModel::mutate()
 {
   array<double> mut_scale(sp_sep * repro_rate * mutation * (tstep - last_mut_tstep));
   last_mut_tstep=tstep;
-  array<unsigned> new_sp;
+  array<unsigned> new_sp, cell_ids;
   array<unsigned> num_new_sp;
   for (auto& i: *this)
     {
-      auto new_species_in_cell=i->as<EcolabCell>()->mutate(mut_scale);
-      num_new_sp<<=new_species_in_cell.size();
-      new_sp <<= new_species_in_cell;
+      new_sp <<= i->as<EcolabCell>()->mutate(mut_scale);
+      cell_ids <<= array<int>(new_sp.size()-cell_ids.size(),i.id());
     }
-  unsigned offset=species.size(), offi=0;
-  // assign 1 for all new species created in this cell, 0 for the others
-  for (auto& i: *this)
+//  unsigned offset=species.size(), offi=0;
+//  // assign 1 for all new species created in this cell, 0 for the others
+//  for (auto& i: *this)
+//    {
+//      auto& density=i->as<EcolabCell>()->density;
+//      density<<=array<int>(new_sp.size(),0);
+//      for (size_t j=0; j<num_new_sp[offi]; ++j)
+//        {
+//          density[j+offset]=1;
+//        }
+//      offset+=num_new_sp[offi++];
+//    }
+#ifdef MPI_SUPPORT
+  MPIbuf b; b<<new_sp<<cell_ids; b.gather(0);
+  if (myid()==0)
     {
-      auto& density=i->as<EcolabCell>()->density;
-      density<<=array<int>(new_sp.size(),0);
-      for (size_t j=0; j<num_new_sp[offi]; ++j)
-        {
-          density[j+offset]=1;
-        }
-      offset+=num_new_sp[offi++];
+      new_sp.resize(0); cell_ids.resize(0);
+      for (unsigned i=0; i<nprocs(); i++) 
+	{
+	  array<int> n,c;
+	  b>>n>>c;
+	  new_sp<<=n; cell_ids<<=c;
+	}
+      ModelData::mutate(new_sp);
     }
+  MPIbuf() << cell_ids << *(ModelData*)this << bcast(0)
+           >> cell_ids >> *(ModelData*)this;
+#else
   ModelData::mutate(new_sp);
+#endif
+  // set the new species density to 1 for those created on this cell
+  for (auto& i: *this)
+    i->as<EcolabCell>()->density <<= cell_ids==i.id();
 }
 
 array<int> EcolabPoint::mutate(const array<double>& mut_scale)
@@ -451,7 +476,7 @@ void SpatialModel::setGrid(size_t nx, size_t ny)
         o->neighbours.push_back(makeId(i,j-1)); 
         o->neighbours.push_back(makeId(i,j+1)); 
       }
-  rebuildPtrLists();
+  partitionObjects();
 }
 
 void SpatialModel::generate(unsigned niter)
@@ -470,8 +495,7 @@ void SpatialModel::migrate()
 
   for (auto& o: objects) o->salt=array_urand.rand();
 
-  // prepareNeighbours
-
+  prepareNeighbours();
   vector<array<int> > delta(size(), array<int>(species.size(),0));
 
   for (size_t i=0; i<size(); i++)
@@ -520,7 +544,7 @@ void SpatialModel::makeConsistent()
   unsigned long nsp=0;
   for (auto& i: *this) nsp=max(nsp,i->as<EcolabCell>()->density.size());
 #ifdef MPI_SUPPORT
-  MPI_AllReduce(MPI_IN_PLACE,&nsp,1,MPI_UNSIGNED_LONG,MPI_MAX,MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE,&nsp,1,MPI_UNSIGNED_LONG,MPI_MAX,MPI_COMM_WORLD);
 #endif
   for (auto& i: *this)
     i->as<EcolabCell>()->density<<=array<int>(nsp-i->as<EcolabCell>()->density.size(),0);

src/pythonMain.cc

@@ -0,0 +1,35 @@
+/*
+  @copyright Russell Standish 2024
+  @author Russell Standish
+  This file is part of EcoLab
+
+  Open source licensed under the MIT license. See LICENSE for details.
+*/
+
+#include <Python.h>
+#include <boost/locale.hpp>
+#ifdef MPI_SUPPORT
+#include "classdescMP.h"
+#endif
+#include "ecolab_epilogue.h"
+
+#include <string>
+#include <vector>
+using namespace std;
+using boost::locale::conv::utf_to_utf;
+
+int main(int argc, char* argv[])
+{
+#ifdef MPI_SUPPORT
+  classdesc::MPISPMD spmd(argc, argv);
+#endif
+  // convert arguments to UTF-16 equivalents
+  // Python 3.8 and later have Py_BytesMain, which obviates this code
+  vector<wstring> wargs;
+  for (int i=0; i<argc; ++i)
+    wargs.push_back(utf_to_utf<wchar_t>(argv[i]));
+  wchar_t* wargsData[argc];
+  for (int i=0; i<argc; ++i)
+    wargsData[i]=const_cast<wchar_t*>(wargs[i].c_str());
+  return Py_Main(argc,wargsData);
+}