Partial checkin of parallel command work.

Makefile

@@ -61,7 +61,7 @@ ifeq ($(OS),Darwin)
 CXXFLAGS+=-std=c++11
 endif
 
-VPATH=include src models
+VPATH+=src
 
 # objects to build ecolab.a
 OBJS=src/automorph.o src/auxil.o src/arrays.o src/sparse_mat.o \
@@ -81,7 +81,7 @@ OBJS+=src/getContext.o
 endif
 
 #CDHDRS=graph.cd graphcode.cd netcomplexity.cd object.cd plot.cd poly.cd polyRESTProcess.cd polyRESTProcessBase.cd ref.cd random.cd random_basic.cd ref.cd RESTProcess_base.cd signature.cd sparse_mat.cd
-CDHDRS=object.cd
+CDHDRS=
 
 # Clunky, but this extracts all .cd files mentioned in header files,
 CDHDRS+=$(shell bash extractCDHeaders.sh)
@@ -156,7 +156,7 @@ ecolab-libs: lib bin
 
 .PHONY: models classdesc
 
-$(OBJS) $(MODS:%=src/%): $(CDHDRS:%=include/%) 
+$(OBJS) $(MODS:%=src/%): $(CDHDRS:%=include/%) graphcode.cd
 
 $(OBJS:.o=.d) $(MODS:%.o=src/%.d):  $(ECOLAB_HOME)/$(MCFG)
 
@@ -254,8 +254,8 @@ latex-docs:
 	if which latex; then cd doc; rm -f *.aux *.dvi *.log *.blg *.toc *.lof *.out; latex -interaction=batchmode ecolab; fi
 
 #bin/ecolab is a python interpreter supporting MPI
-bin/ecolab$(ECOLIBS_EXT): src/ecolab.o src/pythonMain.o lib/libecolab$(ECOLIBS_EXT).a
-	$(LINK) $(FLAGS) src/ecolab.o src/pythonMain.o -Wl,-rpath $(ECOLAB_HOME)/lib $(LIBS) $(shell pkg-config --libs python3) -o $@
+bin/ecolab$(ECOLIBS_EXT): src/pythonMain.o lib/libecolab$(ECOLIBS_EXT).a
+	$(LINK) $(FLAGS) src/pythonMain.o -Wl,-rpath $(ECOLAB_HOME)/lib $(LIBS) $(shell pkg-config --libs python3) -o $@
 	-find . \( -name "*.cc" -o -name "*.h" \) -print |etags -
 
 .PHONY: install

include/Makefile

@@ -47,7 +47,7 @@ include $(ECOLAB_HOME)/include/Makefile.config
 endif
 
 .SUFFIXES: .cd .d .rc $(SUFFIXES)
-VPATH=$(ECOLAB_HOME)/classdesc:$(ECOLAB_HOME)/classdesc/json5_parser/json5_parser:$(ECOLAB_HOME)/graphcode:$(ECOLAB_HOME)/include
+VPATH=$(ECOLAB_HOME)/classdesc $(ECOLAB_HOME)/classdesc/json5_parser/json5_parser $(ECOLAB_HOME)/graphcode $(ECOLAB_HOME)/include
 PATH:=$(ECOLAB_HOME)/bin:$(PATH):$(HOME)/usr/bin
 CLASSDESC=$(ECOLAB_HOME)/bin/classdesc
 PKG_CONFIG_PATH:=$(HOME)/usr/lib/pkgconfig:/usr/local/lib/pkgconfig:/opt/local/lib/pkgconfig:$(PKG_CONFIG_PATH)

include/ecolab.h

@@ -52,14 +52,12 @@ typedef classdesc::string eco_string;
 
 #include "eco_strstream.h"
 
-
 namespace ecolab
 {
   /* these are defined to default values, even if MPI is false */
   /// MPI process ID and number of processes
-  // for now - need to figure out how to launch python interpreters in MPI later.
-  inline unsigned myid() {return 0;}
-  inline unsigned nprocs() {return 1;}
+  unsigned myid();
+  unsigned nprocs();
 
   using classdesc::pack_t;
   using classdesc::unpack_t;

include/tcl++.h

@@ -233,7 +233,7 @@ namespace ecolab
 
   /* these are defined to default values, even if MPI is false */
   /// MPI process ID and number of processes
-  extern unsigned myid, nprocs;
+  //extern unsigned myid, nprocs;
 
   /// semaphore controlling background event process during command
   /// execution. When zero, background events are processed

models/Makefile

@@ -36,7 +36,7 @@ endif
 # This rule uses a header file of object descriptors
 $(MODELS:=.o): %.o: %.cc 
 
-# how to build a model executable
+# how to build a model
 $(MODELS:=.so): %.so: %.o ../lib/libecolab.a
 	$(LINK) $(FLAGS) -shared -Wl,-rpath $(ECOLAB_HOME)/lib $*.o $(LIBS) -o $@
 

models/ecolab_model.cc

@@ -470,12 +470,14 @@ void SpatialModel::setGrid(size_t nx, size_t ny)
     for (size_t j=0; j<numY; ++j)
       {
         auto o=insertObject(makeId(i,j));
+        o.proc(o.id() % nprocs()); // TODO can we get this to work without this.
         // wire up von Neumann neighborhood
         o->neighbours.push_back(makeId(i-1,j));
         o->neighbours.push_back(makeId(i+1,j)); 
         o->neighbours.push_back(makeId(i,j-1)); 
         o->neighbours.push_back(makeId(i,j+1)); 
       }
+  //distributeObjects();
   partitionObjects();
 }
 

src/ecolab.cc

@@ -11,6 +11,9 @@
 #include "cairoSurfaceImage.h"
 #include "plot.h"
 #include "pythonBuffer.h"
+#ifdef MPI_SUPPORT
+#include "graphcode.h"
+#endif
 #include "ecolab_epilogue.h"
 using namespace ecolab;
 using namespace std;
@@ -22,21 +25,158 @@ namespace ecolab
   bool interpExiting=false;
   void interpExitProc(ClientData cd) {}
 
+#ifdef MPI_SUPPORT
+  unsigned myid() {return graphcode::myid();}
+  unsigned nprocs() {return graphcode::nprocs();}
+#else
+  unsigned myid() {return 0;}
+  unsigned nprocs() {return 1;}
+#endif
+
+
   const char* TCL_args::str() 
 #if (TCL_MAJOR_VERSION==8 && TCL_MINOR_VERSION==0)
     {return Tcl_GetStringFromObj(pop_arg(),NULL);}
 #else
     {return Tcl_GetString(pop_arg());}
 #endif
 
+  namespace
+  {
+    /// Python object to implement MPI process control from Python
+    struct Parallel: public CppPyObject
+    {
+      PyObject* target=nullptr;
+      Parallel();
+    };
+
+    PyMethodDef parallelMethods[]={
+      {"exit",(PyCFunction)exit,METH_NOARGS,"Signal workers to knock off"},
+      {nullptr, nullptr, 0, nullptr}
+    };
+
+    struct ParallelType: public PyTypeObject
+    {
+      static PyObject* call(PyObject* self, PyObject* args, PyObject *kwargs)
+      {
+        if (myid()>0)
+          {
+            PyErr_SetString(PyExc_RuntimeError, "Must be called on master process");
+            return nullptr;
+          }
+        if (!self)
+          {
+            PyErr_SetString(PyExc_RuntimeError, "No target object supplied");
+            return nullptr;
+          }
+        if (!PySequence_Check(args) || PySequence_Size(args)<2)
+          {
+            PyErr_SetString(PyExc_RuntimeError, "Incorrect arguments");
+            return nullptr;
+          }
+        auto target=static_cast<Parallel*>(self)->target;
+        if (!target)
+          {
+            PyErr_SetString(PyExc_RuntimeError, "Workers exited");
+            return nullptr;
+          }
+#ifdef MPI_SUPPORT
+        std::string method=PyUnicode_AsUTF8(PySequence_GetItem(args,0));
+        // we need to pickle the arguments and kwargs, so leave argument support until later
+        MPIBuf b; b<<method<<bcast(0);
+#endif
+        return PyObject_Call(PyObject_GetAttr(target,PySequence_GetItem(args,0)),PyTuple_New(0),nullptr);
+      }
+
+      static PyObject* exit(Parallel* self, PyObject*)
+      {
+        if (self->target)
+          {
+#ifdef MPI_SUPPORT
+            MPIBuf b; b<<"return"<<bcast(0);
+#endif
+            self->target=nullptr;
+          }
+        return Py_None; 
+      }
+
+      static int init(PyObject* self, PyObject* args, PyObject*)
+      {
+        if (!PySequence_Check(args) || PySequence_Size(args)<1)
+          {
+            PyErr_SetString(PyExc_RuntimeError, "Incorrect arguments");
+            return -1;
+          }
+        static_cast<Parallel*>(self)->target=PySequence_GetItem(args,0);
+        return 0;
+      }
+
+      static void finalize(PyObject* self) {exit(static_cast<Parallel*>(self),nullptr);}
+
+      ParallelType()
+      {
+        memset(this,0,sizeof(PyTypeObject));
+        Py_INCREF(this);
+        tp_name="Parallel";
+        tp_methods=parallelMethods;
+        tp_call=call;
+        tp_init=init;
+        tp_finalize=finalize;
+        tp_basicsize=sizeof(Parallel);
+        PyType_Ready(this);
+      }
+
+    };
+
+    ParallelType parallelType;
+
+    Parallel::Parallel()
+    {
+      ob_refcnt=1;
+      ob_type=&parallelType;
+
+#ifdef MPI_SUPPORT
+       for (;myid()>0;)
+        {
+          MPIBuf b; b.bcast(0);
+          std::string method; b>>method;
+          if (method=="return") break;
+          // we need to pickle the arguments and kwargs, so leave argument support until later
+          PyObject_Call(PyObject_GetAttrString(target,method.c_str()),Py_Tuple_new(0),nullptr);
+        }
+#endif
+    }
+
+    void registerParallel()
+    {
+      PyModule_AddObject(pythonModule,"Parallel",reinterpret_cast<PyObject*>(&parallelType));
+    }
+
+    CLASSDESC_ADD_FUNCTION(registerParallel);
+
+  }
+
+
   std::string ecolabHome=ECOLAB_HOME;
   CLASSDESC_ADD_GLOBAL(ecolabHome);
   CLASSDESC_ADD_GLOBAL(array_urand);
   CLASSDESC_ADD_GLOBAL(array_grand);
+  CLASSDESC_ADD_FUNCTION(myid);
+  CLASSDESC_ADD_FUNCTION(nprocs);
   CLASSDESC_DECLARE_TYPE(Plot);
-  CLASSDESC_PYTHON_MODULE(ecolab);
 }
 
+//namespace
+//{
+//  unsigned (*myid)()=ecolab::myid;
+//  unsigned (*nprocs)()=ecolab::nprocs;
+//  CLASSDESC_ADD_GLOBAL(myid);
+//  CLASSDESC_ADD_GLOBAL(nprocs);
+//}
+
+CLASSDESC_PYTHON_MODULE(ecolab);
+
+
 // place for initialising any EcoLab extensions to the TCL
 // interpreter, to be called by tkinter's Tk() object
 extern "C" int Ecolab_Init(Tcl_Interp* interp)

src/pythonMain.cc

@@ -10,26 +10,66 @@
 #include <boost/locale.hpp>
 #ifdef MPI_SUPPORT
 #include "classdescMP.h"
+
+void throw_MPI_errors(MPI_Comm * c, int *code, ...)
+{
+  char *errstr=new char[MPI_MAX_ERROR_STRING+1];
+  int length;
+  MPI_Error_string(*code,errstr,&length);
+  errstr[length]='\0';
+  std::runtime_error MPI_err(errstr);
+  delete [] errstr;
+  throw MPI_err;
+}
+
+
+
 #endif
 #include "ecolab_epilogue.h"
 
+#include <iostream>
 #include <string>
 #include <vector>
 using namespace std;
 using boost::locale::conv::utf_to_utf;
 
+struct Python
+{
+  Python() {Py_Initialize();}
+  ~Python() {Py_FinalizeEx();}
+};
+
 int main(int argc, char* argv[])
 {
 #ifdef MPI_SUPPORT
   classdesc::MPISPMD spmd(argc, argv);
+  MPI_Errhandler errhandler;
+#if MPI_VERSION>=2
+  MPI_Comm_create_errhandler((MPI_Comm_errhandler_function*)throw_MPI_errors,&errhandler);
+  MPI_Comm_set_errhandler(MPI_COMM_WORLD,errhandler);
+#else
+  MPI_Errhandler_create((MPI_Handler_function*)throw_MPI_errors,&errhandler);
+  MPI_Errhandler_set(MPI_COMM_WORLD,errhandler);
+#endif
 #endif
-  // convert arguments to UTF-16 equivalents
-  // Python 3.8 and later have Py_BytesMain, which obviates this code
-  vector<wstring> wargs;
-  for (int i=0; i<argc; ++i)
-    wargs.push_back(utf_to_utf<wchar_t>(argv[i]));
-  wchar_t* wargsData[argc];
-  for (int i=0; i<argc; ++i)
-    wargsData[i]=const_cast<wchar_t*>(wargs[i].c_str());
-  return Py_Main(argc,wargsData);
+  if (argc<2) return 0; // nothing to do
+  // this assumes that all processes see the same filesystem, and
+  // potentially the same CWD if the argument is a relative filename
+  FILE* script=fopen(argv[1],"rb");
+  if (!script) {
+    cerr<<"Failed to open: "<<argv[1]<<endl;
+    return 1; // invalid file
+  }
+  Python python;
+  if (auto path=PySys_GetObject("path"))
+    {
+      PyList_Append(path,PyUnicode_FromString("."));
+      PyList_Append(path,PyUnicode_FromString(ECOLAB_HOME"/lib"));
+    }
+  auto pyArgv=PyList_New(0);
+  for (int i=1; i<argc; ++i)
+    PyList_Append(pyArgv, PyUnicode_FromString(argv[i]));
+  PySys_SetObject("argv",pyArgv);
+
+  return PyRun_SimpleFile(script,argv[1]);
 }

test/Makefile

@@ -19,7 +19,8 @@ GDBM=-lgdbm_compat
 endif
 GDBM+=-lgdbm
 
-MODELS=testrefassign test_tclobjref TCL_obj_vector_bool testref testgraph test_boostgraph testCacheDBM testCairo testTCL_objConstT testTCL_objString test_tcl_stl tcl-arrays complex_tcl_args accessor vector-pair
+#MODELS=testrefassign test_tclobjref TCL_obj_vector_bool testref testgraph test_boostgraph testCacheDBM testCairo testTCL_objConstT testTCL_objString test_tcl_stl tcl-arrays complex_tcl_args accessor vector-pair
+MODELS=testParallel
 # vector-pair - doesn't currently compile - see ticket #161
 #EXTRAMODELS=test_tcl_stl.exe tcl-arrays.exe complex_tcl_args.exe
 EXES=test_omp_rw_lock testCheckpointableFile test_netcomplexity test_boostgraph test_ecostrstream posTest testarrays  callTclArgsMethod pangoTest
@@ -48,11 +49,9 @@ endif
 $(MODELS:=.o): %.o: %.cc 
 	$(CPLUSPLUS) -c -g $(FLAGS) $(CXXFLAGS) $<
 
-# how to build a model executable
-$(MODELS): %: %.o
-# remove any spurious directories aegis puts here
-	rm -rf $@
-	$(LINK) $(FLAGS) $*.o $(MODLINK) $(LIBS) -o $@
+# how to build a model 
+$(MODELS:=.so): %.so: %.o ../lib/libecolab.a
+	$(LINK) $(FLAGS) -shared -Wl,-rpath $(ECOLAB_HOME)/lib $*.o $(LIBS) -o $@
 
 # temporary workaround until directories of same name go
 #$(EXTRAMODELS): %.exe: %.o

test/testParallel.cc

@@ -0,0 +1,7 @@
+#include "testParallel.h"
+#include "testParallel.cd"
+
+Test test;
+CLASSDESC_ADD_GLOBAL(test);
+CLASSDESC_PYTHON_MODULE(test);
+

test/testParallel.h

@@ -0,0 +1,10 @@
+#include "ecolab.h"
+struct Test
+{
+  void runTest()
+  {
+#ifdef MPI_SUPPORT
+    MPIBuf()<<ecolab::myid()<<send(0);
+#endif
+  }
+};