feat[profiler,explorer]: add calltree profiling and dependency metadata

chore: add license tags

CMakeLists.txt

@@ -46,6 +46,8 @@ For more information, please refer to https://discopop-project.github.io/discopo
     endif()
 endif()
 
+find_package(Boost REQUIRED)
+
 message(STATUS "Using LLVM version ${LLVM_PACKAGE_VERSION}")
 message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
 list(APPEND CMAKE_MODULE_PATH ${LLVM_CMAKE_DIR})
@@ -93,6 +95,10 @@ file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build
 file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_RTLIB_VERBOSE=\"${DP_RTLIB_VERBOSE}\"\n")
 file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_NUM_WORKERS=\"${DP_NUM_WORKERS}\"\n")
 file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_INTERNAL_TIMER=\"${DP_INTERNAL_TIMER}\"\n")
+file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_CALLTREE_PROFILING=\"${DP_CALLTREE_PROFILING}\"\n")
+file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_MEMORY_REGION_DEALIASING=\"${DP_MEMORY_REGION_DEALIASING}\"\n")
+
+
 
 # print DiscoPoP configuration
 file(READ ${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py DiscoPoP_CONFIGURATION)

discopop_explorer/PEGraphX.py

@@ -142,13 +142,21 @@ class Dependency:
     sink_line: Optional[LineID] = None
     intra_iteration: bool = False
     intra_iteration_level: int = -1
-    metadata_intra_iteration_dep: Optional[List[LineID]] = None
-    metadata_inter_iteration_dep: Optional[List[LineID]] = None
-    metadata_intra_call_dep: Optional[List[LineID]] = None
-    metadata_inter_call_dep: Optional[List[LineID]] = None
+    metadata_intra_iteration_dep: List[LineID]
+    metadata_inter_iteration_dep: List[LineID]
+    metadata_intra_call_dep: List[LineID]
+    metadata_inter_call_dep: List[LineID]
+    metadata_sink_ancestors: List[LineID]
+    metadata_source_ancestors: List[LineID]
 
     def __init__(self, type: EdgeType):
         self.etype = type
+        self.metadata_intra_iteration_dep = []
+        self.metadata_inter_iteration_dep = []
+        self.metadata_intra_call_dep = []
+        self.metadata_inter_call_dep = []
+        self.metadata_sink_ancestors = []
+        self.metadata_source_ancestors = []
 
     def __str__(self):
         return self.var_name if self.var_name is not None else str(self.etype)
@@ -572,30 +580,27 @@ def parse_dependency(dep: DependenceItem) -> Dependency:
     d.var_name = dep.var_name
     d.memory_region = dep.memory_region
     # parse metadata
-    if ";" in dep.metadata:
-        for md in dep.metadata.split(";"):
+    if len(dep.metadata) > 0:
+        for md in dep.metadata.split(" "):
             if len(md) == 0:
                 continue
-            md_type = md[: md.index(":")]
-            md_raw_values = md[md.index(":") + 1 :]
+            # unpack metadata
+            md_type = md[: md.index("[")]
+            md_raw_values = md[md.index("[") + 1 : -1]
             md_values = [tmp for tmp in md_raw_values.split(",") if len(tmp) > 0]
-
-            if md_type == "intra_iteration_dep":
-                if d.metadata_intra_iteration_dep is None:
-                    d.metadata_intra_iteration_dep = []
+            # store metadata
+            if md_type == "IAI":
                 d.metadata_intra_iteration_dep += md_values
-            elif md_type == "inter_iteration_dep":
-                if d.metadata_inter_iteration_dep is None:
-                    d.metadata_inter_iteration_dep = []
+            elif md_type == "IEI":
                 d.metadata_inter_iteration_dep += md_values
-            elif md_type == "intra_call_dep":
-                if d.metadata_intra_call_dep is None:
-                    d.metadata_intra_call_dep = []
+            elif md_type == "IAC":
                 d.metadata_intra_call_dep += md_values
-            elif md_type == "inter_call_dep":
-                if d.metadata_inter_call_dep is None:
-                    d.metadata_inter_call_dep = []
+            elif md_type == "IEC":
                 d.metadata_inter_call_dep += md_values
+            elif md_type == "SINK_ANC":
+                d.metadata_sink_ancestors += md_values
+            elif md_type == "SOURCE_ANC":
+                d.metadata_source_ancestors += md_values
             else:
                 raise ValueError("Unknown metadata type: ", md_type)
     return d

discopop_explorer/parser.py

@@ -12,7 +12,7 @@
 from collections import defaultdict
 from dataclasses import dataclass
 from os.path import abspath, dirname
-from typing import Any, List, Tuple
+from typing import Any, Dict, List, Tuple
 
 from lxml import objectify  # type:ignore
 
@@ -113,6 +113,31 @@ def __map_dummy_nodes(cu_dict):
 def __parse_dep_file(dep_fd, output_path: str) -> Tuple[List[DependenceItem], List[LoopData]]:
     dependencies_list: List[DependenceItem] = []
     loop_data_list: List[LoopData] = []
+
+    # read dependency metadata
+    dependency_metadata_lines = []
+    if os.path.exists(os.path.join(output_path, "dependency_metadata.txt")):
+        with open(os.path.join(output_path, "dependency_metadata.txt"), "r") as dep_metadata_fd:
+            dependency_metadata_lines = dep_metadata_fd.readlines()
+    dependency_metadata: Dict[Tuple[Any, Any, Any, Any, Any], List[str]] = dict()
+    for line in dependency_metadata_lines:
+        line = line.replace("\n", "")
+        split_line = line.split(" ")
+        if split_line[0].startswith("#"):
+            continue
+        type = split_line[0]
+        sink = split_line[1]
+        source = split_line[2]
+        var = split_line[3]
+        AAvar = split_line[4]
+        line_metadata = " ".join(
+            [split_line[5], split_line[6], split_line[7], split_line[8], split_line[9], split_line[10]]
+        )  # IAC, IAI, IEC, IEI, SINK_ANC, SOURCE_ANC)
+        key_tuple = sink, source, type, var, AAvar
+        if key_tuple not in dependency_metadata:
+            dependency_metadata[key_tuple] = []
+        dependency_metadata[key_tuple].append(line_metadata)
+
     # read static dependencies
     static_dependency_lines = []
     if not os.path.exists(os.path.join(output_path, "static_dependencies.txt")):
@@ -155,21 +180,29 @@ def __parse_dep_file(dep_fd, output_path: str) -> Tuple[List[DependenceItem], Li
             var_str = "" if len(source_fields) == 1 else source_fields[1]
             var_name = ""
             aa_var_name = ""
-            metadata = ""
+            metadata = []
             if len(var_str) > 0:
                 if "(" in var_str:
                     split_var_str = var_str.split("(")
                     var_name = split_var_str[0]
                     aa_var_name = split_var_str[1][
                         : split_var_str[1].index(")")
                     ]  # name of the allocated variable which is accessed, i.e. variable name after anti aliasing
-                    metadata_str = split_var_str[1][split_var_str[1].index(")") + 1 :]
-                    if "[" in metadata_str:
-                        metadata = metadata_str[1:-1]
                 else:
                     # compatibility with results created without alias analysis
                     var_name = var_str
-            dependencies_list.append(DependenceItem(sink, source_fields[0], type, var_name, aa_var_name, metadata))
+            # retrieve metadata
+            key_tuple = sink, source_fields[0], type, var_name, aa_var_name
+            if key_tuple in dependency_metadata:
+                metadata = dependency_metadata[key_tuple]
+            # register dependencies
+            if len(metadata) == 0:
+                dependencies_list.append(DependenceItem(sink, source_fields[0], type, var_name, aa_var_name, ""))
+            else:
+                for md_set in metadata:
+                    dependencies_list.append(
+                        DependenceItem(sink, source_fields[0], type, var_name, aa_var_name, md_set)
+                    )
 
     return dependencies_list, loop_data_list
 

discopop_explorer/pattern_detectors/do_all_detector.py

@@ -25,6 +25,7 @@
     MemoryRegion,
     DepType,
     NodeID,
+    Dependency,
 )
 from ..utils import classify_loop_variables, filter_for_hotspots
 from ..variable import Variable
@@ -83,6 +84,8 @@ def run_detection(pet: PEGraphX, hotspots, reduction_info: List[ReductionInfo])
     result: List[DoAllInfo] = []
     nodes = pet.all_nodes(LoopNode)
 
+    warnings.warn("DOALL DETECTION CURRENTLY ASSUMES THE EXISTENCE OF DEPENDENCY METADATA!")
+
     # remove reduction loops
     print("ASDF: ", [r.node_id for r in reduction_info])
     print("Nodes: ", [n.start_position() for n in nodes])
@@ -239,10 +242,6 @@ def __check_loop_dependencies(
         memory_regions_defined_in_loop.update(mem_regs)
 
     for source, target, dep in deps:
-
-        if root_loop.start_position() == "1:153":
-            pass
-
         # todo: move this calculation to the innermost point possible to reduce computation costs
         # get metadata for dependency
         dep_source_nesting_level = __calculate_nesting_level(pet, root_loop, source)
@@ -264,34 +263,81 @@ def __check_loop_dependencies(
         if pet.is_loop_index(dep.var_name, loop_start_lines, root_children_cus):
             continue
 
+        # ignore dependencies where either source or sink do not lie within root_loop
+        if len(dep.metadata_source_ancestors) > 0 and len(dep.metadata_sink_ancestors) > 0:
+            if not (
+                (root_loop.start_position() in dep.metadata_sink_ancestors)
+                and root_loop.start_position() in dep.metadata_source_ancestors
+            ):
+                tmp = root_loop.start_position()
+                continue
+
         # targeted variable is not read-only
         if dep.dtype == DepType.INIT:
             continue
         elif dep.dtype == DepType.RAW:
             # check RAW dependencies
             # RAW problematic, if it is not an intra-iteration RAW
-            if not dep.intra_iteration:
+            cond_1 = (len(dep.metadata_intra_iteration_dep) == 0) and parent_function_lineid in (
+                dep.metadata_intra_call_dep if dep.metadata_intra_call_dep is not None else []
+            )
+            cond_2 = len([cf for cf in called_functions_lineids if cf in dep.metadata_inter_call_dep]) > 0
+            cond_3 = len([t for t in parent_loops if t in dep.metadata_inter_iteration_dep]) > 0
+            if (cond_1) or ((cond_2) and (cond_3)):
                 return True
             # if it is an intra iteration dependency, it is problematic if it belongs to a parent loop
             else:
                 if dep.intra_iteration_level <= max_considered_intra_iteration_dep_level:
                     if pet.node_at(source) in root_children_cus and pet.node_at(target) in root_children_cus:
                         pass
                     else:
-                        return True
+                        # check if metadata exists
+                        if dep.metadata_intra_iteration_dep is not None:
+                            for t in dep.metadata_intra_iteration_dep:
+                                if t in parent_loops:
+                                    return True
+                            return False
+                        else:
+                            return True
 
         elif dep.dtype == DepType.WAR:
             # check WAR dependencies
             # WAR problematic, if it is not an intra-iteration WAR and the variable is not private or firstprivate
-            if not dep.intra_iteration:
+            if (
+                not dep.intra_iteration
+                and (dep.metadata_intra_iteration_dep is None or len(dep.metadata_intra_iteration_dep) == 0)
+                and parent_function_lineid
+                in (dep.metadata_intra_call_dep if dep.metadata_intra_call_dep is not None else [])
+            ) or (
+                (
+                    False
+                    if dep.metadata_inter_call_dep is None
+                    else (len([cf for cf in called_functions_lineids if cf in dep.metadata_inter_call_dep]) > 0)
+                )
+                and (
+                    False
+                    if dep.metadata_inter_iteration_dep is None
+                    else (len([t for t in parent_loops if t in dep.metadata_inter_iteration_dep]) > 0)
+                )
+            ):
                 if dep.var_name not in [v.name for v in first_privates + privates + last_privates]:
                     # check if variable is defined inside loop
                     if dep.memory_region not in memory_regions_defined_in_loop:
                         return True
+                    # check if the definitions of the accessed variable originates from a function call
+                    if __check_for_problematic_function_argument_access(pet, source, target, dep):
+                        return True
             # if it is an intra iteration dependency, it is problematic if it belongs to a parent loop
             elif dep.intra_iteration_level > root_loop.get_nesting_level(pet):
-                tmp = root_loop.get_nesting_level(pet)
-                return True
+                tmp_nesting_level = root_loop.get_nesting_level(pet)
+                # check if metadata exists
+                if len(dep.metadata_intra_iteration_dep) != 0:
+                    for t in dep.metadata_intra_iteration_dep:
+                        if t in parent_loops:
+                            return True
+                    return False
+                else:
+                    return True
         elif dep.dtype == DepType.WAW:
             # check WAW dependencies
             # handled by variable classification
@@ -386,3 +432,35 @@ def __get_called_functions(pet: PEGraphX, root_loop: LoopNode) -> List[LineID]:
 
     # convert node ids of called functions to line ids
     return [pet.node_at(n).start_position() for n in called_functions]
+
+
+def __check_for_problematic_function_argument_access(
+    pet: PEGraphX, source: NodeID, target: NodeID, dep: Dependency
+) -> bool:
+    """duplicates exists: do_all_detector <-> reduction_detector !"""
+    # check if the "same" function argument is accessed and it is a pointer type.
+    # if so, return True. Else. return false
+
+    # find accessed function argument for source
+    source_pf = pet.get_parent_function(pet.node_at(source))
+    source_accessed_pf_args = [a for a in source_pf.args if a.name == dep.var_name]
+    if len(source_accessed_pf_args) == 0:
+        return False
+
+    # find accessed function argument for target
+    target_pf = pet.get_parent_function(pet.node_at(target))
+    target_accessed_pf_args = [a for a in target_pf.args if a.name == dep.var_name]
+    if len(target_accessed_pf_args) == 0:
+        return False
+
+    # check for overlap in accessed args
+
+    for source_a in source_accessed_pf_args:
+        for target_a in target_accessed_pf_args:
+            if source_a == target_a:
+                # found overlap
+                # check for pointer type
+                if "*" in source_a.type:
+                    return True
+    # not problematic
+    return False