From 247787ffb2f371123b8c98ba9c9bb1d6178a0263 Mon Sep 17 00:00:00 2001 From: Katie Graham Date: Sun, 12 Feb 2023 20:47:02 -0800 Subject: [PATCH 01/11] Added option to set path to imagenet data --- applications/vision/data/imagenet/__init__.py | 57 ++++++++++++------- applications/vision/resnet.py | 8 ++- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 636f625daf6..88b3b77fd37 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -5,7 +5,7 @@ import lbann import lbann.contrib.launcher -def make_data_reader(num_classes=1000, small_testing=False): +def make_data_reader(num_classes=1000, small_testing=False, data_path=None): # Load Protobuf message from file current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -18,27 +18,44 @@ def make_data_reader(num_classes=1000, small_testing=False): google.protobuf.text_format.Merge(f.read(), message) message = message.data_reader - # Paths to ImageNet data - # Note: Paths are only known for some compute centers - compute_center = lbann.contrib.launcher.compute_center() - if compute_center == 'lc': - from lbann.contrib.lc.paths import imagenet_dir, imagenet_labels - train_data_dir = imagenet_dir(data_set='train', - num_classes=num_classes) - train_label_file = imagenet_labels(data_set='train', - num_classes=num_classes) - test_data_dir = imagenet_dir(data_set='val', - num_classes=num_classes) - test_label_file = imagenet_labels(data_set='val', + + if data_path != None: + print("Setting up data reader") + train_data_dir = os.path.join(data_path, 'train') + test_data_dir = os.path.join(data_path, "val") + #FIXME: This might not match the directory structure of the user's + # dataset. We could infer the labels from the directory structure, + # but the data reader is expecting a file. + train_label_file = os.path.join(data_path, "labels/train.txt") + test_label_file = os.path.join(data_path, "labels/val.txt") + + elif lbann.contrib.launcher.compute_center() in ['lc', 'nersc']: + # Paths to ImageNet data + # Note: Paths are only known for some compute centers + compute_center = lbann.contrib.launcher.compute_center() + if compute_center == 'lc': + from lbann.contrib.lc.paths import imagenet_dir, imagenet_labels + train_data_dir = imagenet_dir(data_set='train', num_classes=num_classes) - elif compute_center == 'nersc': - from lbann.contrib.nersc.paths import imagenet_dir, imagenet_labels - train_data_dir = imagenet_dir(data_set='train') - train_label_file = imagenet_labels(data_set='train') - test_data_dir = imagenet_dir(data_set='val') - test_label_file = imagenet_labels(data_set='val') + train_label_file = imagenet_labels(data_set='train', + num_classes=num_classes) + test_data_dir = imagenet_dir(data_set='val', + num_classes=num_classes) + test_label_file = imagenet_labels(data_set='val', + num_classes=num_classes) + print("Data paths") + print(train_data_dir) + print(train_label_file) + print(test_data_dir) + print(test_label_file) + elif compute_center == 'nersc': + from lbann.contrib.nersc.paths import imagenet_dir, imagenet_labels + train_data_dir = imagenet_dir(data_set='train') + train_label_file = imagenet_labels(data_set='train') + test_data_dir = imagenet_dir(data_set='val') + test_label_file = imagenet_labels(data_set='val') else: - raise RuntimeError(f'ImageNet data paths are unknown for current compute center ({compute_center})') + raise RuntimeError(f'ImageNet data paths are unknown for current compute center ({compute_center}). Set the "data-path" argument to the location of your dataset.') # Check that data paths are accessible if not os.path.isdir(train_data_dir): diff --git a/applications/vision/resnet.py b/applications/vision/resnet.py index 005204421fe..cba02ef3ee5 100644 --- a/applications/vision/resnet.py +++ b/applications/vision/resnet.py @@ -50,9 +50,14 @@ parser.add_argument( '--random-seed', action='store', default=0, type=int, help='random seed for LBANN RNGs', metavar='NUM') +parser.add_argument( + '--data-path', action='store', default=None, type=str, + help='Path to imagenet directory. default: None') lbann.contrib.args.add_optimizer_arguments(parser, default_learning_rate=0.1) args = parser.parse_args() +#d = __import__(args.datapath) + # Due to a data reader limitation, the actual model realization must be # hardcoded to 1000 labels for ImageNet. imagenet_labels = 1000 @@ -145,7 +150,8 @@ opt = lbann.contrib.args.create_optimizer(args) # Setup data reader -data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes) +data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes, + data_path=args.data_path) # Setup trainer trainer = lbann.Trainer(mini_batch_size=args.mini_batch_size, random_seed=args.random_seed) From 21c7b7bda1abdd6e6372cc9db6f77642078a8fd5 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:23:50 -0800 Subject: [PATCH 02/11] Update applications/vision/resnet.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/resnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vision/resnet.py b/applications/vision/resnet.py index cba02ef3ee5..6c3cab460e3 100644 --- a/applications/vision/resnet.py +++ b/applications/vision/resnet.py @@ -52,7 +52,7 @@ help='random seed for LBANN RNGs', metavar='NUM') parser.add_argument( '--data-path', action='store', default=None, type=str, - help='Path to imagenet directory. default: None') + help='Path to top-level imagenet directory. default: None') lbann.contrib.args.add_optimizer_arguments(parser, default_learning_rate=0.1) args = parser.parse_args() From 9003d9cb666bf9dd036049b4cd3948271041d5f4 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:24:07 -0800 Subject: [PATCH 03/11] Update applications/vision/data/imagenet/__init__.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/data/imagenet/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 88b3b77fd37..161c462e1af 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -26,8 +26,8 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): #FIXME: This might not match the directory structure of the user's # dataset. We could infer the labels from the directory structure, # but the data reader is expecting a file. - train_label_file = os.path.join(data_path, "labels/train.txt") - test_label_file = os.path.join(data_path, "labels/val.txt") + train_label_file = os.path.join(data_path, 'labels/train.txt') + test_label_file = os.path.join(data_path, 'labels/val.txt') elif lbann.contrib.launcher.compute_center() in ['lc', 'nersc']: # Paths to ImageNet data From 5f2e6ef28fe474621bf99916684a614c7c9123b9 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:24:21 -0800 Subject: [PATCH 04/11] Update applications/vision/data/imagenet/__init__.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/data/imagenet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 161c462e1af..0d078862931 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -19,7 +19,7 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): message = message.data_reader - if data_path != None: + if data_path is not None: print("Setting up data reader") train_data_dir = os.path.join(data_path, 'train') test_data_dir = os.path.join(data_path, "val") From 767594d92870bd94969a75809009788d76167058 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:24:55 -0800 Subject: [PATCH 05/11] Update applications/vision/resnet.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/resnet.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/applications/vision/resnet.py b/applications/vision/resnet.py index 6c3cab460e3..bb0a48fdbe7 100644 --- a/applications/vision/resnet.py +++ b/applications/vision/resnet.py @@ -56,8 +56,6 @@ lbann.contrib.args.add_optimizer_arguments(parser, default_learning_rate=0.1) args = parser.parse_args() -#d = __import__(args.datapath) - # Due to a data reader limitation, the actual model realization must be # hardcoded to 1000 labels for ImageNet. imagenet_labels = 1000 From 95c6e7989151a42c38dc0d3b1983c675fa3d6792 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:25:05 -0800 Subject: [PATCH 06/11] Update applications/vision/data/imagenet/__init__.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/data/imagenet/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 0d078862931..f852e62c561 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -43,11 +43,6 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): num_classes=num_classes) test_label_file = imagenet_labels(data_set='val', num_classes=num_classes) - print("Data paths") - print(train_data_dir) - print(train_label_file) - print(test_data_dir) - print(test_label_file) elif compute_center == 'nersc': from lbann.contrib.nersc.paths import imagenet_dir, imagenet_labels train_data_dir = imagenet_dir(data_set='train') From a67af7598a268dafed948ed06aa270e636ca8790 Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:25:18 -0800 Subject: [PATCH 07/11] Update applications/vision/data/imagenet/__init__.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/data/imagenet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index f852e62c561..87698c87b67 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -22,7 +22,7 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): if data_path is not None: print("Setting up data reader") train_data_dir = os.path.join(data_path, 'train') - test_data_dir = os.path.join(data_path, "val") + test_data_dir = os.path.join(data_path, 'val') #FIXME: This might not match the directory structure of the user's # dataset. We could infer the labels from the directory structure, # but the data reader is expecting a file. From 8d3a81286b2939176b1b6c539ec4ae49755a513f Mon Sep 17 00:00:00 2001 From: Katie Graham <50850420+graham63@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:25:49 -0800 Subject: [PATCH 08/11] Update applications/vision/data/imagenet/__init__.py Co-authored-by: Tom Benson <30674819+benson31@users.noreply.github.com> --- applications/vision/data/imagenet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 87698c87b67..82e7e5b832d 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -50,7 +50,7 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): test_data_dir = imagenet_dir(data_set='val') test_label_file = imagenet_labels(data_set='val') else: - raise RuntimeError(f'ImageNet data paths are unknown for current compute center ({compute_center}). Set the "data-path" argument to the location of your dataset.') + raise RuntimeError(f'ImageNet data paths are unknown for current compute center ({compute_center}). Set "--data-path" to the location of your dataset.') # Check that data paths are accessible if not os.path.isdir(train_data_dir): From 9bf2931d725c7cb4cb494f236ba088f0d29cca4d Mon Sep 17 00:00:00 2001 From: Katie Graham Date: Tue, 14 Feb 2023 15:00:29 -0800 Subject: [PATCH 09/11] Removed resolved FIXME note --- applications/vision/data/imagenet/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/applications/vision/data/imagenet/__init__.py b/applications/vision/data/imagenet/__init__.py index 82e7e5b832d..c6f05da0883 100644 --- a/applications/vision/data/imagenet/__init__.py +++ b/applications/vision/data/imagenet/__init__.py @@ -23,9 +23,6 @@ def make_data_reader(num_classes=1000, small_testing=False, data_path=None): print("Setting up data reader") train_data_dir = os.path.join(data_path, 'train') test_data_dir = os.path.join(data_path, 'val') - #FIXME: This might not match the directory structure of the user's - # dataset. We could infer the labels from the directory structure, - # but the data reader is expecting a file. train_label_file = os.path.join(data_path, 'labels/train.txt') test_label_file = os.path.join(data_path, 'labels/val.txt') From 739254284464a04eb287c7fd85934d1d4ebd250e Mon Sep 17 00:00:00 2001 From: Katie Graham Date: Sun, 26 Feb 2023 20:39:13 -0800 Subject: [PATCH 10/11] Added note about --data-path in .rst docs --- docs/data_ingestion.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/data_ingestion.rst b/docs/data_ingestion.rst index 1f29fbee33c..71b96438ae0 100644 --- a/docs/data_ingestion.rst +++ b/docs/data_ingestion.rst @@ -1,3 +1,8 @@ +.. role:: bash(code) + :language: bash +.. role:: python(code) + :language: python + Data Ingestion ============== @@ -27,6 +32,14 @@ Legacy Data Readers Some of the legacy data readers are the ``MNIST``, ``ImageNet``, and ``CIFAR10`` data readers. +.. note:: The imagenet data reader uses a path that may not be known + to all compute centers. If the dataset is not found + :python:`--data-path` may be set to the top level of the data + set in :code:`resnet.py`, :code:`alexnet.py`, and + :code:`densenet.py`. The data set is must contain + :code:`labels/train.txt`, :code:`labels/val.txt`, + :code:`train/`, and :code:`val/`. + "New" Data Readers ------------------- From 5fbe41e04ec7008e4a365b9e3445dee00df2d808 Mon Sep 17 00:00:00 2001 From: Katie Graham Date: Mon, 6 Mar 2023 09:57:50 -0800 Subject: [PATCH 11/11] Added --data-path to densenet and alexnet --- applications/vision/alexnet.py | 6 +++++- applications/vision/densenet.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/applications/vision/alexnet.py b/applications/vision/alexnet.py index b67f2673714..fab9fbc36a6 100644 --- a/applications/vision/alexnet.py +++ b/applications/vision/alexnet.py @@ -22,6 +22,9 @@ parser.add_argument( '--num-classes', action='store', default=1000, type=int, help='number of ImageNet classes (default: 1000)', metavar='NUM') +parser.add_argument( + '--data-path', action='store', default=None, type=str, + help='Path to top-level imagenet directory. default: None') lbann.contrib.args.add_optimizer_arguments(parser) args = parser.parse_args() @@ -64,7 +67,8 @@ opt = lbann.contrib.args.create_optimizer(args) # Setup data reader -data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes) +data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes, + data_path=args.data_path) # Setup trainer trainer = lbann.Trainer(mini_batch_size=args.mini_batch_size) diff --git a/applications/vision/densenet.py b/applications/vision/densenet.py index 5e27bdfd517..aae8b25d52f 100644 --- a/applications/vision/densenet.py +++ b/applications/vision/densenet.py @@ -428,6 +428,8 @@ def get_args(): parser.add_argument("--print-matrix-summary", dest="print_matrix_summary", action="store_const", const=True, default=False) + parser.add_argument('--data-path', action='store', default=None, type=str, + help='Path to top-level imagenet directory. default: None') args = parser.parse_args() return args @@ -438,7 +440,7 @@ def set_up_experiment(args, labels): algo = lbann.BatchedIterativeOptimizer("sgd", epoch_count=args.num_epochs) - + # Set up objective function cross_entropy = lbann.CrossEntropy([probs, labels]) layers = list(lbann.traverse_layer_graph(input_)) @@ -472,7 +474,9 @@ def set_up_experiment(args, callbacks=callbacks) # Set up data reader - data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes, small_testing=True) + data_reader = data.imagenet.make_data_reader(num_classes=args.num_classes, + small_testing=True, + data_path=args.data_path) percentage = 0.001 * 2 * (args.mini_batch_size / 16) * 2