Merge pull request #514 from m-a-d-n-e-s-s/evaleev/feature/worldobj-f…

…uture-trace

added ability to "trace" (count) futures associated with WorldObjects

CMakeLists.txt

@@ -249,6 +249,12 @@ add_feature_info(TASK_DEBUG_TRACE ENABLE_TASK_DEBUG_TRACE "supports debug trace
 set(MADNESS_TASK_DEBUG_TRACE ${ENABLE_TASK_DEBUG_TRACE} CACHE BOOL
         "Enable task debug tracing.")
 
+option(ENABLE_WORLDOBJECT_FUTURE_TRACE
+        "Enable tracing of futures assicuated with WorldObjects." OFF)
+add_feature_info(WORLDOBJECT_FUTURE_TRACE ENABLE_WORLDOBJECT_FUTURE_TRACE "supports tracing of futures associated with WorldObjects")
+set(MADNESS_WORLDOBJECT_FUTURE_TRACE ${ENABLE_WORLDOBJECT_FUTURE_TRACE} CACHE BOOL
+        "Enable tracing of futures assicuated with WorldObjects.")
+
 set(FORTRAN_INTEGER_SIZE 4 CACHE STRING "The fortran integer size (4 or 8 bytes) used for BLAS and LAPACK function calls")
 if(NOT (FORTRAN_INTEGER_SIZE EQUAL 4 OR FORTRAN_INTEGER_SIZE EQUAL 8))
   message(FATAL_ERROR "Incorrect fortran integer size '${FORTRAN_INTEGER_SIZE}'\n"

cmake/config.h.in

@@ -126,6 +126,9 @@
 #cmakedefine MADNESS_DQ_USE_PREBUF 1
 #cmakedefine MADNESS_DQ_PREBUF_SIZE @MADNESS_DQ_PREBUF_SIZE@
 #cmakedefine MADNESS_ASSUMES_ASLR_DISABLED 1
+#cmakedefine MADNESS_WORLDOBJECT_FUTURE_TRACE 1
+#cmakedefine MADNESS_WORLDOBJECT_FUTURE_TRACE_WORLD_ID @MADNESS_WORLDOBJECT_FUTURE_TRACE_WORLD_ID@
+#cmakedefine MADNESS_WORLDOBJECT_FUTURE_TRACE_MAX_NOBJECTS @MADNESS_WORLDOBJECT_FUTURE_TRACE_MAX_NOBJECTS@
 
 /* Define to the equivalent of the C99 'restrict' keyword, or to
    nothing if this is not supported.  Do not define if restrict is

src/madness/world/test_world.cc

@@ -495,8 +495,21 @@ void test6(World& world) {
         a.task(p, &Foo::ping, me, 1);
       }
 
+#ifdef MADNESS_WORLDOBJECT_FUTURE_TRACE
+      for (ProcessID p = 0; p != nproc; ++p) {
+        auto f = a.task(p, &Foo::get0);
+        a.trace(f);
+      }
+#endif
+
       world.gop.fence();
 
+#ifdef MADNESS_WORLDOBJECT_FUTURE_TRACE
+      MADNESS_CHECK(a.trace_status_nfuture_registered() == a.trace_futures() ? nproc : 0);
+      MADNESS_CHECK(decltype(a)::trace_status_nfuture_assigned(a.id()) ==
+                    decltype(a)::trace_futures(a.id()) ? nproc : 0);
+#endif
+
       // stress the large message protocol ... off by default
       if (0) {
         const auto dbuf_sum_long =
@@ -1349,9 +1362,7 @@ int main(int argc, char** argv) {
 #if  MADNESS_CATCH_SIGNALS
     signal(SIGSEGV, mad_signal_handler);
 #endif
-    initialize(argc,argv);
-
-    World world(SafeMPI::COMM_WORLD);
+    World& world = initialize(argc,argv);
 
     redirectio(world);
     print("The processor frequency is",cpu_frequency());

src/madness/world/world_object.h

@@ -38,10 +38,14 @@
 #ifndef MADNESS_WORLD_WORLD_OBJECT_H__INCLUDED
 #define MADNESS_WORLD_WORLD_OBJECT_H__INCLUDED
 
-#include <type_traits>
 #include <madness/world/thread.h>
 #include <madness/world/world_task_queue.h>
 
+#include <array>
+#include <atomic>
+#include <cstddef>
+#include <type_traits>
+
 /// \addtogroup world_object
 /// @{
 
@@ -1380,8 +1384,168 @@ namespace madness {
               world.unregister_ptr(this->id());
             }
         }
+
+#ifdef MADNESS_WORLDOBJECT_FUTURE_TRACE
+        /// "traces" future evaluation by counting their assignments in a static table
+
+        /// Counts future assignments in a a statically-sized table to make this as lightweight/lock-free as possible
+        /// with minimal effort. Can only trace objects of a single World.
+        /// \param[in,out] f the future to be traced; if ready, will be unchanged (but contribute to the trace
+        /// statistics of this object), or have a callback registered that will update the tracing statistics on
+        /// assignment
+        /// \warning this function will trace futures for WorldObjects associated with default world (id=0) only;
+        /// use CMake variable `MADNESS_WORLDOBJECT_FUTURE_TRACE_WORLD_ID` to adjust the target World ID.
+        /// \warning this function will trace futures for WorldObjects with IDs < 1000000 only;
+        /// use CMake variable `MADNESS_WORLDOBJECT_FUTURE_TRACE_MAX_NOBJECTS` to adjust the limit.
+        template <typename T>
+        std::enable_if_t<!std::is_same_v<T,void>,void> trace(Future<T>& f) const;
+
+        /// \param[in] id a WorldObject ID
+        /// \return true if futures associated with \p id are traced
+        static bool trace_futures(const uniqueidT &id);
+
+        /// \return true if futures associated with this object are traced
+        bool trace_futures() const {
+          return trace_futures(this->id());
+        }
+
+        /// \param[in] id report tracing stats for this WorldObject
+        /// \return number of futures given to trace() of the WorldObject with ID \p id
+        static std::size_t trace_status_nfuture_registered(const uniqueidT& id);
+
+        /// \return number of futures given to trace() of this object
+        std::size_t trace_status_nfuture_registered() const {
+          return trace_status_nfuture_registered(this->id());
+        }
+
+        /// \param[in] id report tracing stats for this WorldObject
+        /// \return number of assigned futures given to trace() of the WorldObject with ID \p id
+        static std::size_t trace_status_nfuture_assigned(const uniqueidT& id);
+
+        /// \return number of assigned futures registered via `this->trace()`
+        std::size_t trace_status_nfuture_assigned() const {
+          return trace_status_nfuture_assigned(this->id());
+        }
+#endif
     };
 
+#ifdef MADNESS_WORLDOBJECT_FUTURE_TRACE
+    namespace detail {
+    template <typename Derived> struct WorldObjectFutureTracer {
+      // this value is the world ID to trace
+      constexpr static std::size_t world_id =
+#ifndef MADNESS_WORLDOBJECT_FUTURE_TRACE_WORLD_ID
+          0
+#else
+          MADNESS_WORLDOBJECT_FUTURE_TRACE_WORLD_ID
+#endif
+          ;
+      // this value is 1 greater than is the highest ID of WorldObjects to trace
+      constexpr static std::size_t max_object_id =
+#ifndef MADNESS_WORLDOBJECT_FUTURE_TRACE_MAX_NOBJECTS
+          1000000
+#else
+          MADNESS_WORLDOBJECT_FUTURE_TRACE_MAX_NOBJECTS
+#endif
+          ;
+
+      static constexpr bool do_trace(const uniqueidT& id) {
+        return id.get_world_id() == world_id &&
+               id.get_obj_id() < max_object_id;
+      }
+
+      static std::array<std::atomic<std::size_t>, max_object_id>
+          nfuture_registered;
+      static std::array<std::atomic<std::size_t>, max_object_id>
+          nfuture_assigned;
+
+      struct Initializer {
+        Initializer() {
+          for (auto &&v : nfuture_registered) {
+            v.store(0);
+          }
+          for (auto &&v : nfuture_assigned) {
+            v.store(0);
+          }
+        }
+      };
+      static Initializer initializer;
+
+      struct FutureTracer : public CallbackInterface {
+        FutureTracer(const uniqueidT &id) : id_(id) {
+          if (do_trace(id_)) {
+            nfuture_registered[id_.get_obj_id()]++;
+          }
+        }
+
+        // Not allowed
+        FutureTracer(const FutureTracer &) = delete;
+        FutureTracer &operator=(const FutureTracer &) = delete;
+
+        virtual ~FutureTracer() {}
+
+        /// Notify this object that the future has been set.
+
+        /// This will set the value of the future on the remote node and delete
+        /// this callback object.
+        void notify() override {
+          if (do_trace(id_)) {
+            nfuture_assigned[id_.get_obj_id()]++;
+          }
+          delete this;
+        }
+
+      private:
+        uniqueidT id_;
+      }; // struct FutureTracer
+
+    }; // struct WorldObjectFutureTracer
+    template <typename Derived>
+    typename WorldObjectFutureTracer<Derived>::Initializer
+        WorldObjectFutureTracer<Derived>::initializer;
+    template <typename Derived>
+    std::array<std::atomic<std::size_t>, WorldObjectFutureTracer<Derived>::max_object_id>
+        WorldObjectFutureTracer<Derived>::nfuture_registered;
+    template <typename Derived>
+    std::array<std::atomic<std::size_t>, WorldObjectFutureTracer<Derived>::max_object_id>
+        WorldObjectFutureTracer<Derived>::nfuture_assigned;
+    }  // namespace detail
+
+    template <typename Derived>
+    template <typename T>
+    std::enable_if_t<!std::is_same_v<T,void>,void> WorldObject<Derived>::trace(Future<T>& f) const {
+      f.register_callback(
+          new typename detail::WorldObjectFutureTracer<Derived>::FutureTracer(
+              this->id()));
+    }
+
+    template <typename Derived>
+    bool WorldObject<Derived>::trace_futures(const uniqueidT& id) {
+      return detail::WorldObjectFutureTracer<Derived>::do_trace(id);
+    }
+
+    template <typename Derived>
+    std::size_t WorldObject<Derived>::trace_status_nfuture_registered(const uniqueidT& id) {
+      if (detail::WorldObjectFutureTracer<
+              Derived>::do_trace(id)) {
+        return detail::WorldObjectFutureTracer<
+            Derived>::nfuture_registered[id.get_obj_id()];
+      }
+      else return 0;
+    }
+
+    template <typename Derived>
+    std::size_t WorldObject<Derived>::trace_status_nfuture_assigned(const uniqueidT& id) {
+      if (detail::WorldObjectFutureTracer<
+              Derived>::do_trace(id)) {
+        return detail::WorldObjectFutureTracer<
+            Derived>::nfuture_assigned[id.get_obj_id()];
+      }
+      else return 0;
+    }
+
+#endif  // MADNESS_WORLDOBJECT_FUTURE_TRACE
+
     namespace archive {
 
         /// Specialization of \c ArchiveLoadImpl for globally-addressable objects.
@@ -1461,8 +1625,8 @@ namespace madness {
                 ar & ptr->id();
             }
         };
-    }
-}
+    }   // namespace archive
+}  // namespace madness
 
 #endif // MADNESS_WORLD_WORLD_OBJECT_H__INCLUDED