From 4eda0b2c886e62b5e66459eb0bf866b4dd63c2a0 Mon Sep 17 00:00:00 2001
From: Artem Polyakov <artpol84@gmail.com>
Date: Wed, 26 Jun 2019 14:43:16 -0700
Subject: [PATCH] Fix the callsite report

Use the MPI operation name as an indicator of the starting point of
a callsite.
We know that MPI_<op> is located in the wrappers.c (or maybe in another
profiler's wrapper file).
So we only output callsite starting from frames above the one, containing
MPI_<op>

Signed-off-by: Artem Polyakov <artpol84@gmail.com>
---
 mpiPi.h.in |  4 +++-
 report.c   | 36 ++++++++++++++++++++++++++----------
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/mpiPi.h.in b/mpiPi.h.in
index 82f0789..3c73bd1 100644
--- a/mpiPi.h.in
+++ b/mpiPi.h.in
@@ -46,7 +46,9 @@
 
 #define MPIP_HELP_LIST "@PACKAGE_BUGREPORT@"
 
-#define MPIP_CALLSITE_STACK_DEPTH (mpiPi.stackDepth)
+#define MPIP_CALLSITE_STACK_INT_ADDITION 2
+#define MPIP_CALLSITE_STACK_DEPTH \
+  (mpiPi.stackDepth + MPIP_CALLSITE_STACK_INT_ADDITION)
 #define MPIP_CALLSITE_STATS_COOKIE 518641
 #define MPIP_CALLSITE_STATS_COOKIE_ASSERT(f) {assert(MPIP_CALLSITE_STATS_COOKIE==((f)->cookie));}
 
diff --git a/report.c b/report.c
index 045f100..57b68dd 100644
--- a/report.c
+++ b/report.c
@@ -583,38 +583,54 @@ mpiPi_print_callsites (FILE * fp)
 
   for (i = 0; i < ac; i++)
     {
-      int j;
-      for (j = 0, stack_continue_flag = 1;
+      int j, start_point = 0;
+      char *mpi_funcname = mpiPi.lookup[av[i]->op - mpiPi_BASE].name;
+
+      /* Find the starting point:
+       * search for the MPI operation name in the stacktrace and use it
+       * as the beginning of the stack.
+       */
+      for (j = 0;
+           (j < MPIP_CALLSITE_STACK_DEPTH) && (av[i]->filename[j] != NULL);
+           j++)
+        {
+
+          if (strcmp (av[i]->functname[j], mpi_funcname) == 0)
+            {
+              start_point = j + 1;
+            }
+        }
+
+      for (j = start_point, stack_continue_flag = 1;
            (j < MPIP_CALLSITE_STACK_DEPTH) && (av[i]->filename[j] != NULL) &&
            stack_continue_flag == 1; j++)
         {
+          int level = j - start_point;
           if (av[i]->line[j] == 0 &&
               (strcmp (av[i]->filename[j], "[unknown]") == 0 ||
                strcmp (av[i]->functname[j], "[unknown]") == 0))
             {
               fprintf (fp, "%3d %3d %-*s %-*s %s\n",
                        av[i]->id,
-                       j,
+                       level,
                        fileLenMax + 6,
                        mpiP_format_address (av[i]->pc[j], addr_buf),
                        funcLenMax,
                        av[i]->functname[j],
-                       (j ==
-                        0) ? &(mpiPi.lookup[av[i]->op -
-                             mpiPi_BASE].name[4]) : "");
+                       (level == 0) ?
+                         &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]) : "");
             }
           else
             {
               fprintf (fp, "%3d %3d %-*s %5d %-*s %s\n",
                        av[i]->id,
-                       j,
+                       level,
                        fileLenMax,
                        av[i]->filename[j], av[i]->line[j],
                        funcLenMax,
                        av[i]->functname[j],
-                       (j ==
-                        0) ? &(mpiPi.lookup[av[i]->op -
-                             mpiPi_BASE].name[4]) : "");
+                       (level == 0) ?
+                         &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]) : "");
             }
           /*  Do not bother printing stack frames above main   */
           if (strcmp (av[i]->functname[j], "main") == 0