diff --git a/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py b/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py
index 3b670dc464..a0d4fabb77 100644
--- a/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py
+++ b/python/raft-ann-bench/src/raft-ann-bench/run/__main__.py
@@ -239,7 +239,6 @@ def main():
         )
     conf_filename = conf_filepath.split("/")[-1]
     conf_filedir = "/".join(conf_filepath.split("/")[:-1])
-    dataset_name = conf_filename.replace(".json", "")
     dataset_path = args.dataset_path
     if not os.path.exists(conf_filepath):
         raise FileNotFoundError(conf_filename)
@@ -247,6 +246,8 @@ def main():
     with open(conf_filepath, "r") as f:
         conf_file = json.load(f)
 
+    dataset_name = conf_file["dataset"]["name"]
+
     executables_to_run = dict()
     # At least one named index should exist in config file
     if args.indices: