Merge pull request #41 from RWTH-ACS/cleanup-ne

Cleanup

.gitlab-ci.yml

@@ -41,20 +41,6 @@ prepare:centos8:cuda11:
   tags:
     - docker
 
-# check if styleguide is fulfilled
-#style_check:
-#  stage: build
-#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
-#  allow_failure: true
-#  tags:
-#    - docker
-#  script:
-#    - ls
-#    - utils/test_style.sh
-
-# Stage: build
-##############################################################################
-
 build:
   stage: build
   needs: ["prepare:rocky9:docker-dev"]
@@ -173,12 +159,15 @@ build:debug:
     PARAMETER: ''
     CHDIR: 'tests'
   script:
-    - mkdir ~/.ssh &&
-      echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa &&
-      echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa &&
-      echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa &&
-      echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub &&
-      echo $KNOWN_HOSTS > ~/.ssh/known_hosts && chmod 600 ~/.ssh/id_rsa
+    - mkdir ~/.ssh
+    - echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa
+    - echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa
+    - echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa
+    - echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub
+    - echo $KNOWN_HOSTS > ~/.ssh/known_hosts
+    - chmod 600 ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/known_hosts
+    - chmod 600 ~/.ssh/id_rsa.pub
     - ssh $GPU_TARGET mkdir -p $RDIR
     - scp -r $LDIR/* $GPU_TARGET:$RDIR/
     - ssh $GPU_TARGET "LD_PRELOAD=$RDIR/libtirpc.so.3 $RDIR/cricket-rpc-server 255" &

Makefile

@@ -1,8 +1,8 @@
 #MIT License...
 
-.PHONY: all cuda-gdb libtirpc gpu cpu tests clean install install-cpu bin/tests
+.PHONY: all cuda-gdb libtirpc gpu cpu tests clean install install-cpu
 
-all: gpu cpu install
+all: cpu install
 
 clean:
 	@echo -e "\033[31m----> Cleaning up gpu\033[0m"
@@ -36,27 +36,28 @@ tests:
 install-cpu: bin/cricket-client.so bin/cricket-rpc-server bin/libtirpc.so bin/libtirpc.so.3 bin/tests
 	@echo -e "\033[36m----> Copying cpu binaries to build/bin\033[0m"
 
-install: install-cpu bin/cricket
+install-gpu: bin/cricket
+	@echo -e "\033[36m----> Copying gpu binaries to build/bin\033[0m"
+
+install: install-cpu
 	@echo -e "\033[36m----> Copying to build/bin\033[0m"
 
 bin:
 	mkdir bin
 
 bin/tests: bin tests
-	ln -s ../tests/bin bin/tests
+	ln -sf ../tests/bin bin/tests
 
-bin/cricket-client.so: bin
-	$(MAKE) -C cpu cricket-client.so
+bin/cricket-client.so: bin cpu
 	cp cpu/cricket-client.so bin
 
 bin/cricket-server.so: bin
 	$(MAKE) -C cpu cricket-server.so
-	mv cpu/cricket-server.so bin/cricket-server.so
+	cp cpu/cricket-server.so bin/cricket-server.so
 
 
-bin/cricket-rpc-server: bin
-	$(MAKE) -C cpu cricket-rpc-server
-	mv cpu/cricket-rpc-server bin/cricket-rpc-server
+bin/cricket-rpc-server: bin cpu
+	cp cpu/cricket-rpc-server bin/cricket-rpc-server
 
 bin/cricket: bin gpu
 	cp gpu/cricket bin

cpu/Makefile

@@ -42,7 +42,7 @@ SRC_SERVER = $(RPC_XDR)                 \
 			 mt-memcpy.c				\
 			 cpu-elf2.c					\
 			 cpu-server-nvml.c			\
-			 cpu-server-cudnn.c  \
+			 cpu-server-cudnn.c         \
 			 cpu-server-cublaslt.c
 
 SRC_SERVER_LIB = server-library.c
@@ -63,11 +63,9 @@ SRC_CLIENT = $(RPC_XDR)                 \
 			 cpu-elf2.c					\
 			 cpu-client-nvml.c          \
 			 cpu-client-cudnn.c			\
-			 cpu-client-cublas.c  \
+			 cpu-client-cublas.c        \
 			 cpu-client-cublaslt.c
 
-# 			 cpu-client-driver-hidden.c \
-
 ifdef WITH_IB
 SRC_SERVER += cpu-ib.c
 SRC_CLIENT += cpu-ib.c

cpu/cpu-elf2.c

@@ -954,11 +954,20 @@ int elf2_parameter_info(list *kernel_infos, void* memory, size_t memsize)
             LOGE(LOG_ERROR, "gelf_getsym failed for entry %d", entry->kernel_id);
             continue;
         }
+
         if ((kernel_str = elf_strptr(elf, symtab_shdr.sh_link, sym.st_name) ) == NULL) {
             LOGE(LOG_ERROR, "strptr failed for entry %d", entry->kernel_id);
             continue;
         }
 
+        /* When using (some?) intrinsics, nvcc adds symbols for them in the .nv.info table.
+        * They are prefixed with $__internal_7_$ and are not kernels. We skip them he
+        */
+        const char *intrinsics_prefix = "$__internal_";
+        if (strncmp(kernel_str, intrinsics_prefix, strlen(intrinsics_prefix)) == 0) {
+            continue;
+        }
+
         if (utils_search_info(kernel_infos, kernel_str) != NULL) {
             continue;
         }

cpu/cpu-server-driver.c

@@ -312,7 +312,7 @@ bool_t rpc_cumoduleloaddata_1_svc(mem_data mem, ptr_result *result,
         LOGE(LOG_ERROR, "error in resource manager");
     }
     if (result->err != 0) {
-        char *err_str = NULL;
+        const char *err_str = NULL;
         cuGetErrorName(result->err, &err_str);
         LOGE(LOG_DEBUG, "cuModuleLoadData result: %s", err_str);
     }
@@ -332,7 +332,7 @@ bool_t rpc_cumoduleload_1_svc(char* path, ptr_result *result,
         LOGE(LOG_ERROR, "error in resource manager");
     }
     if (result->err != 0) {
-        char *err_str = NULL;
+        const char *err_str = NULL;
         cuGetErrorName(result->err, &err_str);
         LOGE(LOG_DEBUG, "cuModuleLoad result: %s", err_str);
     }

gpu/src/main.c

@@ -150,6 +150,24 @@ int cricket_restore(int argc, char *argv[])
     uint64_t warp_mask;
     size_t jmptbl_len;
 
+    uint32_t lanemask;
+    uint64_t sswarps;
+    cricket_jmptable_index *index;
+    cricket_jmptable_index *kernelindex;
+    uint64_t relative_ssy;
+    uint64_t jmptbl_address;
+    const char *fn;
+    uint32_t predicate = 1;
+    uint64_t cur_address = 0;
+    uint64_t start_address = 0;
+    uint64_t jmptable_addr;
+    uint64_t rb_address;
+    bool found_callstack = false;
+    uint32_t numDev = 0;
+
+
+    CricketDeviceProp dev_prop;
+
 #ifdef CRICKET_PROFILE
     //vars for profiling
     double bt, ct, dt, et, ft, gt, comt;
@@ -246,7 +264,6 @@ int cricket_restore(int argc, char *argv[])
     LOGE(LOG_DEBUG, "got CUDA debugging API");
 
     // We currently only support a single GPU
-    uint32_t numDev = 0;
     if (!cricket_device_get_num(cudbgAPI, &numDev)) {
         LOGE(LOG_ERROR, "error getting device num");
         goto detach;
@@ -257,7 +274,7 @@ int cricket_restore(int argc, char *argv[])
 
     // Get device/architecture data so we know how many SMs/warps/lanes
     // to restore
-    CricketDeviceProp dev_prop;
+
     if (!cricket_device_get_prop(cudbgAPI, 0, &dev_prop)) {
         printf("error getting device properties\n");
         goto detach;
@@ -284,22 +301,8 @@ int cricket_restore(int argc, char *argv[])
 
     // cricket_cr_kernel_name(cudbgAPI, 0,0,0, &kernel_name);
     // printf("cricket: kernel-name: \"%s\"\n", kernel_name);
-
-    uint32_t lanemask;
-    uint64_t sswarps;
     warp_info.dev = 0;
     warp_info.dev_prop = &dev_prop;
-    cricket_jmptable_index *index;
-    cricket_jmptable_index *kernelindex;
-    uint64_t relative_ssy;
-    uint64_t jmptbl_address;
-    const char *fn;
-    uint32_t predicate = 1;
-    uint64_t cur_address = 0;
-    uint64_t start_address = 0;
-    uint64_t jmptable_addr;
-    uint64_t rb_address;
-    bool found_callstack = false;
 
     // We first need to navigate the jumptable in the kernel entry
     // function.
@@ -804,7 +807,7 @@ int cricket_restore(int argc, char *argv[])
 
 int cricket_checkpoint(int argc, char *argv[])
 {
-    char *ckp_dir = "/tmp/cricket-ckp";
+    const char *ckp_dir = "/tmp/cricket-ckp";
     uint32_t numDev = 0;
     const char *kernel_name = NULL;
     const char *warp_kn;
@@ -818,6 +821,8 @@ int cricket_checkpoint(int argc, char *argv[])
     int ret = -1;
     CUDBGResult res;
     CUDBGAPI cudbgAPI;
+    cricketWarpInfo warp_info = { 0 };
+
 #ifdef CRICKET_PROFILE
     struct timeval a, b, c, d, e, f;
     struct timeval la, lb, lc, ld, le, lf, lg;
@@ -875,7 +880,7 @@ int cricket_checkpoint(int argc, char *argv[])
     }
     cricket_device_print_prop(&dev_prop);
 
-    warp_mask = malloc(sizeof(uint64_t) * dev_prop.numSMs);
+    warp_mask = (uint64_t*)malloc(sizeof(uint64_t) * dev_prop.numSMs);
 
     res = cudbgAPI->readValidWarps(0, 0, warp_mask);
     if (res != CUDBG_SUCCESS) {
@@ -912,7 +917,6 @@ int cricket_checkpoint(int argc, char *argv[])
            elf_info.stack_size, elf_info.param_addr, elf_info.param_size,
            elf_info.param_num);
 
-    cricketWarpInfo warp_info = { 0 };
     warp_info.dev = 0;
     warp_info.dev_prop = &dev_prop;
     warp_info.sm = 0;

tests/test_apps/Makefile

@@ -23,7 +23,7 @@ OVERHEAD_BIN = overhead.testapp
 OVERHEAD_O = overhead.o
 
 TEST_KERNEL_LIB_O = test_kernel_lib.o
-TEST_KERNEL_LIB = test_kernel.so
+TEST_KERNEL_LIB = libtest_kernel.so
 TEST_KERNEL_LIB_CALL_O = test_kernel_call.o
 TEST_KERNEL_LIB_CALL = test_kernel_call
 
@@ -73,7 +73,7 @@ $(TEST_KERNEL_LIB_O) : $(FILES)
 	$(CC) $(CFLAGS) $(DEBUG_FLAGS) -dc --compiler-options '-fPIC' -o $@ $<
 
 $(TEST_KERNEL_LIB) : $(TEST_KERNEL_LIB_O)
-	$(LD) $(LDFLAGS) $(DEBUG_FLAGS) -shared -o lib$@ $^
+	$(LD) $(LDFLAGS) $(DEBUG_FLAGS) -shared -o $@ $^
 
 $(TEST_KERNEL_LIB_CALL_O) : $(TEST_KERNEL_LIB_CALL_O:.o=.c)
 	$(HOST_CC) -c -o $@ $<

utils/Dockerfile

@@ -26,7 +26,7 @@ RUN dnf install -y make bash git gcc autoconf libtool automake rpcgen \
 ENV LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:${LD_LIBRARY_PATH}"
 
 RUN dnf -y config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && \
-    dnf --refresh -y install cuda-compiler-12-1 cuda-libraries-devel-12-1 cuda-driver-devel-12-1 cuda-profiler-api-12-1 cuda-nvml-devel-12-1 nvidia-driver-NVML-530.30.02 libcudnn8-devel && \
+    dnf --refresh -y install cuda-compiler-12-1 cuda-libraries-devel-12-1 cuda-driver-devel-12-1 cuda-profiler-api-12-1 cuda-nvml-devel-12-1 nvidia-driver-NVML libcudnn8-devel && \
     ln -s cuda-12.1 /usr/local/cuda && \
     ln -s libcuda.so /usr/local/cuda/targets/x86_64-linux/lib/stubs/libcuda.so.1