Allow for randomized sampling

common/kokkos-sampler/README.md

@@ -1,4 +1,4 @@
-This is a sampler utility that is intended to complement other tools in the Kokkos Tools set. This utility allows for sampling (rather than collecting) of profiling or debugging data gathered from a particular tool of the Kokkos Tools set. The Kokkos Tools user provides a sampling rate via the environment variable KOKKOS_TOOLS_SAMPLER_SKIP.  
+This is a sampler utility that is intended to complement other tools in the Kokkos Tools set. This utility allows for sampling (rather than collecting) of profiling or debugging data gathered from a particular tool of the Kokkos Tools set. The Kokkos Tools user provides a sampling rate via the environment variable KOKKOS_TOOLS_SAMPLER_SKIP. It also has an environment variable for sampling probability, KOKKOS_TOOLS_SAMPLER_PROBABILITY that is a percent, taken as a float, between 0.0 and 100.0.
 
 In order for the state of the sampled profiling and logging data in memory to be captured at the time of the utility's callback invocation, it might be important to enforce fences. However, this also means that there are more synchronization points compared with running the program without the tool.
 This fencing behavior can be controlled by setting the environment variable `KOKKOS_TOOLS_GLOBALFENCES`. A non-zero value implies global fences on invocation of the tool. The default is not to introduce extra fences.	

common/kokkos-sampler/kp_sampler_skip.cpp

@@ -5,13 +5,17 @@
 #include <dlfcn.h>
 #include "../../profiling/all/kp_core.hpp"
 #include "kp_config.hpp"
+#include <ctime>  // for random number generation
 
 namespace KokkosTools {
 namespace Sampler {
-static uint64_t uniqID           = 0;
-static uint64_t kernelSampleSkip = 101;
-static int tool_verbosity        = 0;
-static int tool_globFence        = 0;
+static uint64_t uniqID = 0;
+static uint64_t kernelSampleSkip =
+    101;  // Default skip rate of every 100 invocations
+static float tool_prob_num =
+    1.0;  // Default probability of 1 percent of all invocations
+static int tool_verbosity = 0;
+static int tool_globFence = 0;
 
 typedef void (*initFunction)(const int, const uint64_t, const uint32_t, void*);
 typedef void (*finalizeFunction)();
@@ -40,6 +44,7 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
                           const uint32_t devInfoCount, void* deviceInfo) {
   const char* tool_verbose_str   = getenv("KOKKOS_TOOLS_SAMPLER_VERBOSE");
   const char* tool_globFence_str = getenv("KOKKOS_TOOLS_GLOBALFENCES");
+
   if (NULL != tool_verbose_str) {
     tool_verbosity = atoi(tool_verbose_str);
   } else {
@@ -54,15 +59,15 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
   char* profileLibrary = getenv("KOKKOS_TOOLS_LIBS");
   if (NULL == profileLibrary) {
     printf(
-        "Checking KOKKOS_PROFILE_LIBRARY. WARNING: This is a depreciated "
-        "variable. Please use KOKKOS_TOOLS_LIBS\n");
+        "KokkosP: Checking KOKKOS_PROFILE_LIBRARY. WARNING: This is a "
+        "deprecated "
+        "variable. Please use KOKKOS_TOOLS_LIBS. \n");
     profileLibrary = getenv("KOKKOS_PROFILE_LIBRARY");
     if (NULL == profileLibrary) {
-      printf("KokkosP: No library to call in %s\n", profileLibrary);
+      printf("KokkosP: No library to call in %s.\n", profileLibrary);
       exit(-1);
     }
   }
-
   char* envBuffer = (char*)malloc(sizeof(char) * (strlen(profileLibrary) + 1));
   strcpy(envBuffer, profileLibrary);
 
@@ -96,14 +101,12 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
           (beginFunction)dlsym(childLibrary, "kokkosp_begin_parallel_scan");
       beginReduceCallee =
           (beginFunction)dlsym(childLibrary, "kokkosp_begin_parallel_reduce");
-
       endScanCallee =
           (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_scan");
       endForCallee =
           (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_for");
       endReduceCallee =
           (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_reduce");
-
       initProfileLibrary =
           (initFunction)dlsym(childLibrary, "kokkosp_init_library");
       finalizeProfileLibrary =
@@ -136,109 +139,203 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
 
   uniqID = 1;
 
-  const char* tool_sample = getenv("KOKKOS_TOOLS_SAMPLER_SKIP");
+  const char* tool_sample      = getenv("KOKKOS_TOOLS_SAMPLER_SKIP");
+  const char* tool_probability = getenv("KOKKOS_TOOLS_SAMPLER_PROBABILITY");
+  // composing the periodicity and probability parameter. Set to 1 if
+  // probability for each periodic sample. Set to 0 to default to probability
+  // even if periodicity (skip rate) is defined.
+
   if (NULL != tool_sample) {
     kernelSampleSkip = atoi(tool_sample) + 1;
   }
 
+  if (NULL != tool_probability) {
+    //  read sampling probability as an float between 0 and 100, representing
+    //  a percentage that data should be gathered.
+    //  Connector reasons about probability as a double between 0.0 and 1.0.
+    tool_prob_num = atof(tool_probability);
+    if (tool_prob_num > 100.0) {
+      printf(
+          "KokkosP: The sampling probability value is set to be greater than "
+          "100.0. "
+          "Setting sampling probability to 100 percent; all of the "
+          "invocations of a Kokkos Kernel will be profiled.\n");
+      tool_prob_num = 100.0;
+    } else if (tool_prob_num < 0.0) {
+      printf(
+          "KokkosP: The sampling probability value is set to be negative "
+          "number. Setting "
+          "sampling probability to 0 percent; none of the invocations of "
+          "a Kokkos Kernel will be profiled.\n");
+      tool_prob_num = 0.0;
+    }
+  };
+
   if (tool_verbosity > 0) {
     printf("KokkosP: Sampling rate set to: %s\n", tool_sample);
+    printf("KokkosP: Sampling probability set to: %s\n", tool_probability);
+    printf(
+        "KokkosP: seeding Random Number Generator using clock for "
+        "probabilistic sampling.\n");
   }
-}
+  srand(time(NULL));
+
+  if ((NULL != tool_probability) && (NULL != tool_sample)) {
+    printf(
+        "KokkosP: Note that both probability and skip rate are set. The Kokkos "
+        "Tools Sampler utility will invoke a Kokkos Tool child event you "
+        "specified "
+        "(e.g., the profiler or debugger tool connector you specified "
+        "in KOKKOS_TOOLS_LIBS) with the specified sampling probability applied "
+        "to the "
+        "specified sampling skip rate set.\n");
+  }
+}  // end kokkosp_init_library
 
 void kokkosp_finalize_library() {
   if (NULL != finalizeProfileLibrary) (*finalizeProfileLibrary)();
 }
 
 void kokkosp_begin_parallel_for(const char* name, const uint32_t devID,
                                 uint64_t* kID) {
-  *kID = uniqID++;
+  *kID = 0;
 
-  if (((*kID) % kernelSampleSkip) == 0) {
-    if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-begin function...\n",
-             (unsigned long long)(*kID));
-    }
+  static uint64_t invocationNum;
+  ++invocationNum;
+  if ((invocationNum % kernelSampleSkip) == 0) {
+    if ((rand() / (1.0 * RAND_MAX)) < (tool_prob_num / 100.0)) {
+      *kID = 1;
 
-    if (NULL != beginForCallee) {
-      (*beginForCallee)(name, devID, kID);
+      if (tool_verbosity > 0) {
+        printf(
+            "KokkosP: sample %llu of parallel_for on its invocation number %d "
+            "calling"
+            "child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
+      }
+      if (NULL != beginForCallee) {
+        (*beginForCallee)(name, devID, kID);
+      }
+      if (tool_verbosity > 1)
+        printf(
+            "KokkosP: sample for a parallel_for with kernel ID %llu on its "
+            "invocation number "
+            "%d called child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
     }
-  }
-}
+  }  // end sampling
+
+}  // kokkosp_begin_parallel_for
 
 void kokkosp_end_parallel_for(const uint64_t kID) {
-  if ((kID % kernelSampleSkip) == 0) {
+  if (kID >
+      0) {  // the corresponding kokkosp_begin_parallel_for gathered a sample
     if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-end function...\n",
-             (unsigned long long)(kID));
+      printf(
+          "KokkosP: sample for a parallel_for with kernel ID %llu calling "
+          "child-end "
+          "function...\n",
+          (unsigned long long)(kID));
     }
 
     if (NULL != endForCallee) {
       (*endForCallee)(kID);
     }
-  }
-}
 
-void kokkosp_begin_parallel_scan(const char* name, const uint32_t devID,
-                                 uint64_t* kID) {
-  *kID = uniqID++;
+  }  // end kID > 0
 
-  if (((*kID) % kernelSampleSkip) == 0) {
-    if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-begin function...\n",
-             (unsigned long long)(*kID));
-    }
+}  // kokkosp_end_parallel_for
 
-    if (NULL != beginScanCallee) {
-      (*beginScanCallee)(name, devID, kID);
+void kokkosp_begin_parallel_scan(const char* name, const uint32_t devID,
+                                 uint64_t* kID) {
+  *kID = 0;
+  static uint64_t invocationNum;
+  ++invocationNum;
+  if ((invocationNum % kernelSampleSkip) == 0) {
+    if (rand() / (1.0 * RAND_MAX) < (tool_prob_num / 100.0)) {
+      *kID = 1;  // set kernel ID to 1 so that it is matched with the end.
+      if (tool_verbosity > 0) {
+        printf(
+            "KokkosP: sample %llu for parallel_scan on its invocation number "
+            "%d calling "
+            "child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
+      }
+      if (NULL != beginScanCallee) {
+        (*beginScanCallee)(name, devID, kID);
+      }
+      if (tool_verbosity > 0) {
+        printf(
+            "KokkosP: sample for parallel_scan with kernelID %llu on its "
+            "invocation number %d "
+            "called child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
+      }
     }
-  }
-}
+  }  // end sampling
+}  // kokkosp_begin_parallel_scan
 
 void kokkosp_end_parallel_scan(const uint64_t kID) {
-  if ((kID % kernelSampleSkip) == 0) {
+  if (kID > 0) {
     if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-end function...\n",
-             (unsigned long long)(kID));
+      printf(
+          "KokkosP: sample %llu (a parallel_scan) calling child-end "
+          "function...\n",
+          (unsigned long long)(kID));
     }
-
     if (NULL != endScanCallee) {
       (*endScanCallee)(kID);
     }
   }
-}
+}  // kokkosp_end_parallel_scan
 
 void kokkosp_begin_parallel_reduce(const char* name, const uint32_t devID,
                                    uint64_t* kID) {
-  *kID = uniqID++;
+  *kID = 0;
+  static uint64_t invocationNum;
+  ++invocationNum;
+  if ((invocationNum % kernelSampleSkip) == 0) {
+    if ((rand() / (1.0 * RAND_MAX)) < tool_prob_num / 100.0) {
+      if (tool_verbosity > 0) {
+        printf(
+            "KokkosP: sample %llu for a parallel_reduce on its invocation "
+            "number "
+            "%d calling child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
+      }
 
-  if (((*kID) % kernelSampleSkip) == 0) {
-    if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-begin function...\n",
-             (unsigned long long)(*kID));
-    }
+      *kID = 1;  // set kernel ID to 1 so that it is matched with the end.
+      if (NULL != beginReduceCallee) {
+        (*beginReduceCallee)(name, devID, kID);
+      }
 
-    if (NULL != beginReduceCallee) {
-      (*beginReduceCallee)(name, devID, kID);
+      if (tool_verbosity > 1) {
+        printf(
+            "KokkosP: sample for parallel_reduce with kID %llu on its "
+            "invocation number "
+            "%d called child-begin function...\n",
+            (unsigned long long)(*kID), (int)invocationNum);
+      }
     }
-  }
-}
+  }  // end sampling
+}  // kokkosp_begin_parallel_reduce
 
 void kokkosp_end_parallel_reduce(const uint64_t kID) {
-  if ((kID % kernelSampleSkip) == 0) {
+  if (kID > 0) {
     if (tool_verbosity > 0) {
-      printf("KokkosP: sample %llu calling child-end function...\n",
-             (unsigned long long)(kID));
+      printf(
+          "KokkosP: sample %llu (a parallel_reduce) calling child-end "
+          "function...\n",
+          (unsigned long long)(kID));
     }
-
     if (NULL != endReduceCallee) {
       (*endReduceCallee)(kID);
     }
   }
-}
+}  // kokkosp_end_parallel_reduce
 
 }  // namespace Sampler
-}  // end namespace KokkosTools
+}  // namespace KokkosTools
 
 extern "C" {