From 84f9b4ad4b426db10bfb1b2f4d3c9c93d3b46601 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Thu, 18 Aug 2022 15:15:40 -0400 Subject: [PATCH 1/8] Update ADAM preprocessing to properly open files --- adam_preprocessing/shape_stroke_graph_inference.py | 2 +- adam_preprocessing/shape_stroke_graph_learner.py | 3 +-- adam_preprocessing/utils.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/adam_preprocessing/shape_stroke_graph_inference.py b/adam_preprocessing/shape_stroke_graph_inference.py index ac0c390f8..b2fe13fec 100644 --- a/adam_preprocessing/shape_stroke_graph_inference.py +++ b/adam_preprocessing/shape_stroke_graph_inference.py @@ -137,7 +137,7 @@ def main(): if len(feature_yamls) == 1: # Load features, update them, then save with open( - input_situation_dir / feature_yamls[0], encoding="utf-8" + feature_yamls[0], encoding="utf-8" ) as feature_yaml_in: features = yaml.safe_load(feature_yaml_in) diff --git a/adam_preprocessing/shape_stroke_graph_learner.py b/adam_preprocessing/shape_stroke_graph_learner.py index 06f1bdb2f..e5ee79191 100644 --- a/adam_preprocessing/shape_stroke_graph_learner.py +++ b/adam_preprocessing/shape_stroke_graph_learner.py @@ -161,8 +161,7 @@ def main(): logging.info( "{}, loss: {:.4e}, acc: {}, test acc :{}".format( step, train_loss.item(), acc, test_acc - ), - flush=True, + ) ) logging.info("Best test acc is {}".format(best_acc)) logging.info(f"Saving model state dict to {args.save_model_to}") diff --git a/adam_preprocessing/utils.py b/adam_preprocessing/utils.py index d507a6b0e..23ab778e8 100644 --- a/adam_preprocessing/utils.py +++ b/adam_preprocessing/utils.py @@ -66,7 +66,7 @@ def get_stroke_data( feature_yamls = sorted(situation_dir.glob("feature*")) if len(feature_yamls) == 1: with open( - situation_dir / feature_yamls[0], encoding="utf-8" + feature_yamls[0], encoding="utf-8" ) as feature_yaml_in: features = yaml.safe_load(feature_yaml_in) From f4945a6590b215e3beb8c7874c855c2cf0e42bb6 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Tue, 5 Jul 2022 14:49:08 -0400 Subject: [PATCH 2/8] Add script to generate confusion matrix from GNN decode --- scripts/GNN_vs_expected.py | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 scripts/GNN_vs_expected.py diff --git a/scripts/GNN_vs_expected.py b/scripts/GNN_vs_expected.py new file mode 100644 index 000000000..a7e8036dd --- /dev/null +++ b/scripts/GNN_vs_expected.py @@ -0,0 +1,84 @@ +import argparse +import logging +from os import makedirs +from pathlib import Path +from shutil import rmtree, copytree +from typing import Mapping, Sequence + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sn +import yaml + +def parse_in_domain_objects(input_dir: Path) -> Sequence: + in_domain_objects = set() + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'description.yaml', encoding="utf-8") as description_file: + description_yaml = yaml.safe_load(description_file) + object = description_yaml['language'].rsplit()[-1] + in_domain_objects.add(object) + return sorted(in_domain_objects) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Get confusion matrix for GNN decode" + ) + parser.add_argument('--input-dir', type=Path, help="Curriculum directory to read GNN decodes from") + parser.add_argument('--output-dir', type=Path, help="Directory to output confusion matrix and associated data to") + parser.add_argument( + "-f", "--force", + action='store_true', + required=False, + help='Force overwrite of target directory. By default, the script exits with an error if there already exists ' + 'a directory at the target destination.' + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + input_dir: Path = args.input_dir + output_dir: Path = args.output_dir + force_overwrite: bool = args.force + + if not input_dir.exists(): + logging.warning("Input directory does not exist") + raise FileNotFoundError(str(input_dir)) + + if output_dir.is_dir(): + if not force_overwrite: + logging.warning("There already exists a directory in the target location") + raise FileExistsError(str(output_dir)) + else: + makedirs(output_dir) + + in_domain_objects = parse_in_domain_objects(input_dir) + confusion_matrix = [[0 for _ in range(len(in_domain_objects))] for _ in range(len(in_domain_objects))] + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'description.yaml', encoding="utf-8") as description_file: + description_yaml = yaml.safe_load(description_file) + expected_object = description_yaml['language'].rsplit()[-1] + with open(feature_file_path / 'feature.yaml', encoding="utf-8") as feature_file: + feature_yaml = yaml.safe_load(feature_file) + gnn_object = feature_yaml['objects'][0]['stroke_graph']['concept_name'] + if gnn_object == 'small_single_mug': gnn_object = 'mug' + print(expected_object, gnn_object) + confusion_matrix[in_domain_objects.index(expected_object)][in_domain_objects.index(gnn_object)] += 1 + + df = pd.DataFrame(confusion_matrix, in_domain_objects, in_domain_objects) + sn.set(font_scale=0.9) + sn.color_palette('colorblind') + sn.set_context('paper') + heatmap = sn.heatmap(df, annot=True, cbar=False) + heatmap.set_xticklabels(labels=in_domain_objects, rotation=45, horizontalalignment='right') + plt.xlabel('GNN Object Prediction') + plt.ylabel('Expected Object Label') + plt.title('GNN Decode vs Expected Label', size=18) + plt.tight_layout() + + plt.savefig(output_dir / 'gnn_vs_description.png') + df.to_csv(output_dir / 'gnn_vs_description.csv') + + +if __name__ == '__main__': + main() From 8cb8373c542ddee79b57cf3902da828bc70c0364 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Tue, 12 Jul 2022 13:33:20 -0400 Subject: [PATCH 3/8] Add script to compare adam decode to expected label --- scripts/adam_decode_vs_expected.py | 92 ++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/adam_decode_vs_expected.py diff --git a/scripts/adam_decode_vs_expected.py b/scripts/adam_decode_vs_expected.py new file mode 100644 index 000000000..c670a24df --- /dev/null +++ b/scripts/adam_decode_vs_expected.py @@ -0,0 +1,92 @@ +import argparse +import logging +from os import makedirs +from pathlib import Path +from shutil import rmtree, copytree +from typing import Mapping, Sequence + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sn +import yaml + +def parse_in_domain_objects(input_dir: Path) -> Sequence: + in_domain_objects = set() + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'post_decode.yaml', encoding="utf-8") as description_file: + description_yaml = yaml.safe_load(description_file) + object = description_yaml['gold_language'].rsplit()[-1] + in_domain_objects.add(object) + return sorted(in_domain_objects) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Get confusion matrix for GNN decode" + ) + parser.add_argument('--input-dir', type=Path, help="Curriculum directory with ADAM decode files to read from") + parser.add_argument('--output-dir', type=Path, help="Target directory to output confusion matrix and associated data to") + parser.add_argument( + "-f", "--force", + action='store_true', + required=False, + help='Force overwrite of target directory. By default, the script exits with an error if there already exists ' + 'a directory at the target destination.' + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + input_dir: Path = args.input_dir + output_dir: Path = args.output_dir + force_overwrite: bool = args.force + + if not input_dir.exists(): + logging.warning("Input directory does not exist") + raise FileNotFoundError(str(input_dir)) + + if output_dir.is_dir(): + if not force_overwrite: + logging.warning("There already exists a directory in the target location") + raise FileExistsError(str(output_dir)) + else: + makedirs(output_dir) + + in_domain_objects=parse_in_domain_objects(input_dir) + confusion_matrix = [[0 for _ in range(len(in_domain_objects) + 1)] for _ in range(len(in_domain_objects))] + none_count = 0 + correct_count = 0 + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'post_decode.yaml', encoding="utf-8") as decode_file: + decode_yaml = yaml.safe_load(decode_file) + expected_object = decode_yaml['gold_language'].rsplit()[-1] + if len(decode_yaml['output_language']) > 0: + gnn_object = decode_yaml['output_language'][0]['raw_text'].rsplit()[-1] + print(expected_object, gnn_object) + confusion_matrix[in_domain_objects.index(expected_object)][in_domain_objects.index(gnn_object)] += 1 + if expected_object == gnn_object: + correct_count += 1 + else: + none_count += 1 + confusion_matrix[in_domain_objects.index(expected_object)][-1] += 1 + + + df = pd.DataFrame(confusion_matrix, in_domain_objects, in_domain_objects + ['No Label']) + sn.set(font_scale=0.9) + sn.color_palette('colorblind') + sn.set_context('paper') + heatmap = sn.heatmap(df, annot=True, cbar=False) + heatmap.set_xticklabels(labels=in_domain_objects + ['No Label'], rotation=45, horizontalalignment='right') + plt.xlabel('ADAM Output Language') + plt.ylabel('Expected Object Label') + plt.title('Adam Decode vs Expected Label', size=18) + plt.tight_layout() + + plt.savefig(output_dir / 'adam_decode_vs_description.png') + df.to_csv(output_dir / 'adam_decode_vs_description.csv') + print("Correctly labeled:", correct_count) + print("% Correct:", correct_count/sum(sum(i) for i in arr)) + print("Objects with no label:", none_count) + +if __name__ == '__main__': + main() From 3dc7af194f80d5ba20496ab696f6c245cf26b45d Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Wed, 3 Aug 2022 17:16:03 -0400 Subject: [PATCH 4/8] Add functionality for multiple GNN decodes Note that the confidence score is not set on a per-decode basis here since we currently do not alter confidence values with GNN decodes in ADAM. --- adam/perception/visual_perception.py | 9 +++++++ .../shape_stroke_graph_inference.py | 24 +++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/adam/perception/visual_perception.py b/adam/perception/visual_perception.py index eaaaffed4..11fceda51 100644 --- a/adam/perception/visual_perception.py +++ b/adam/perception/visual_perception.py @@ -156,6 +156,15 @@ def from_mapping( weight=strokes_map["confidence_score"], ) ) + if "concept_names" in strokes_map: + for concept_name in strokes_map["concept_names"]: + properties.append( + StrokeGNNRecognitionNode( + object_recognized=concept_name, + confidence=strokes_map["confidence_score"], + weight=strokes_map["confidence_score"], + ) + ) # Cluster ID is cleaned so that only the digit ID is displayed and not 'object' clusters.append( diff --git a/adam_preprocessing/shape_stroke_graph_inference.py b/adam_preprocessing/shape_stroke_graph_inference.py index b2fe13fec..3ee1cba99 100644 --- a/adam_preprocessing/shape_stroke_graph_inference.py +++ b/adam_preprocessing/shape_stroke_graph_inference.py @@ -4,6 +4,7 @@ from copy import deepcopy import logging from pathlib import Path +from typing import Sequence import torch import torch.nn as nn @@ -22,12 +23,19 @@ from utils import accuracy, get_stroke_data, LinearModel, load_data, STRING_OBJECT_LABELS -def update_features_yaml(features_yaml, *, predicted_object: str): +def update_features_yaml(features_yaml, *, predicted_objects: Sequence[str]): # jac: not terribly efficient to do a full deepcopy, but who cares, this should be a small dict # anyway... also this is probably much less expensive than the GNN inference itself. result = deepcopy(features_yaml) for object_ in result["objects"]: - object_["stroke_graph"]["concept_name"] = predicted_object + if len(predicted_objects) == 1: + object_["stroke_graph"]["concept_name"] = predicted_objects[0] + if "concept_names" in object_["stroke_graph"]: + del object_["stroke_graph"]["concept_names"] + else: + object_["stroke_graph"]["concept_names"] = predicted_objects + if "concept_name" in object_["stroke_graph"]: + del object_["stroke_graph"]["concept_name"] return result @@ -50,6 +58,12 @@ def main(): help="Directory where we should write the feature outputs to. Outputs are structured as if " "the output path is a curriculum directory.", ) + parser.add_argument( + "--top_k", + type=int, + default=1, + help="Top k decodes to retrieve from the GNN", + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO) @@ -129,7 +143,7 @@ def main(): assert outputs.size(0) == curriculum_params["num_dirs"] - predicted_label_ints = outputs.argmax(dim=1) + _, predicted_label_ints = outputs.topk(args.top_k) n_saved = 0 for situation_num in range(curriculum_params["num_dirs"]): input_situation_dir = args.curriculum_path / f"situation_{situation_num}" @@ -143,8 +157,8 @@ def main(): updated_features = update_features_yaml( features, - predicted_object=STRING_OBJECT_LABELS[ - predicted_label_ints[situation_num] + predicted_object=[STRING_OBJECT_LABELS[ + predicted_label_ints[situation_num][i]] for i in range(args.top_k) ], ) From 9f5995415783de5baf89a55ee6b6463a79995777 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Thu, 11 Aug 2022 15:14:26 -0400 Subject: [PATCH 5/8] Add script to aggregate average top GNN decodes. --- scripts/GNN_topk_analysis.py | 88 ++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 scripts/GNN_topk_analysis.py diff --git a/scripts/GNN_topk_analysis.py b/scripts/GNN_topk_analysis.py new file mode 100644 index 000000000..9d2d3a787 --- /dev/null +++ b/scripts/GNN_topk_analysis.py @@ -0,0 +1,88 @@ +import argparse +import logging +from os import makedirs +from pathlib import Path +from shutil import rmtree, copytree +from typing import Mapping, Sequence + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sn +import yaml + +def parse_in_domain_objects(input_dir: Path) -> Sequence: + in_domain_objects = set() + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'description.yaml', encoding="utf-8") as description_file: + description_yaml = yaml.safe_load(description_file) + object = description_yaml['language'].rsplit()[-1] + in_domain_objects.add(object) + return sorted(in_domain_objects) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Get confusion matrix for GNN decode" + ) + parser.add_argument('--input-dir', type=Path, help="Inpuut curriculum directory to read GNN decodes from") + parser.add_argument('--output-dir', type=Path, help="Output directory to write top GNN decode matrix and data to") + parser.add_argument( + "-f", "--force", + action='store_true', + required=False, + help='Force overwrite of target directory. By default, the script exits with an error if there already exists ' + 'a directory at the target destination.' + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + input_dir: Path = args.input_dir + output_dir: Path = args.output_dir + force_overwrite: bool = args.force + + if not input_dir.exists(): + logging.warning("Input directory does not exist") + raise FileNotFoundError(str(input_dir)) + + if output_dir.is_dir(): + if not force_overwrite: + logging.warning("There already exists a directory in the target location") + raise FileExistsError(str(output_dir)) + else: + makedirs(output_dir) + + in_domain_objects = parse_in_domain_objects(input_dir) + object_counts = {in_domain_object: 0 for in_domain_object in in_domain_objects} + gnn_rank_matrix = [[0 for _ in range(len(in_domain_objects))] for _ in range(len(in_domain_objects))] + for feature_file_path in sorted(input_dir.glob("situation*")): + with open(feature_file_path / 'description.yaml', encoding="utf-8") as description_file: + description_yaml = yaml.safe_load(description_file) + expected_object = description_yaml['language'].rsplit()[-1] + object_counts[expected_object] += 1 + with open(feature_file_path / 'feature.yaml', encoding="utf-8") as feature_file: + feature_yaml = yaml.safe_load(feature_file) + gnn_objects = feature_yaml['objects'][0]['stroke_graph']['concept_names'] + for i in range(len(gnn_objects)): + gnn_object = gnn_objects[i] + if gnn_object == 'small_single_mug': gnn_object = 'mug' + gnn_rank_matrix[in_domain_objects.index(expected_object)][in_domain_objects.index(gnn_object)] += len(gnn_objects) - i + for object_id in range(len(gnn_rank_matrix)): + gnn_rank_matrix[object_id] = [label_count / object_counts[in_domain_objects[object_id]] for label_count in arr[object_id]] + df = pd.DataFrame(gnn_rank_matrix, in_domain_objects, in_domain_objects) + sn.set(font_scale=0.9) + sn.color_palette('colorblind') + sn.set_context('paper') + heatmap = sn.heatmap(df, annot=True, cbar=False) + heatmap.set_xticklabels(labels=in_domain_objects, rotation=45, horizontalalignment='right') + plt.xlabel('GNN Object Label Weight') + plt.ylabel('True Object Label') + plt.title('Top GNN Predictions per Object', size=18) + plt.tight_layout() + + plt.savefig(output_dir / 'top_gnn_vs_description.png') + df.to_csv(output_dir / 'top_gnn_vs_description.csv') + + +if __name__ == '__main__': + main() From f11862da36c738a02d0161d1f32b2a70dbb569f8 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Tue, 5 Jul 2022 12:51:10 -0400 Subject: [PATCH 6/8] Add parameter files for downsampling experiments --- adam/curriculum/curriculum_from_files.py | 6 ++ ...m5_objects_v0_with_mugs_full_subset.params | 60 +++++++++++++++++++ ...jects_v0_with_mugs_10pertype_subset.params | 60 +++++++++++++++++++ ...bjects_v0_with_mugs_2pertype_subset.params | 60 +++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 parameters/experiments/p3/m5_objects_v0_with_mugs_full_subset.params create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_10pertype_subset.params create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_2pertype_subset.params diff --git a/adam/curriculum/curriculum_from_files.py b/adam/curriculum/curriculum_from_files.py index f31e003ac..f4e25060e 100644 --- a/adam/curriculum/curriculum_from_files.py +++ b/adam/curriculum/curriculum_from_files.py @@ -30,6 +30,9 @@ "m5_objects_v0_with_mugs", "m5_actions", "m5_actions_person_only", + "m6_objects_downsampled_2pertype_post_gnn", + "m6_objects_downsampled_100 pertype_post_gnn", + "m5_objects_v0_with_mugs_post_gnn", ] PHASE_3_TESTING_CURRICULUM_OPTIONS = [ @@ -40,6 +43,9 @@ "m5_objects_v0_with_mugs_eval", "m5_actions_eval", "m5_actions_person_only_eval", + "m5_objects_v0_with_mugs_eval_2pertype_post_gnn", + "m5_objects_v0_with_mugs_eval_10pertype_post_gnn", + "m5_objects_v0_with_mugs_eval_post_gnn", ] TRAINING_CUR = "training" diff --git a/parameters/experiments/p3/m5_objects_v0_with_mugs_full_subset.params b/parameters/experiments/p3/m5_objects_v0_with_mugs_full_subset.params new file mode 100644 index 000000000..0842ed1d8 --- /dev/null +++ b/parameters/experiments/p3/m5_objects_v0_with_mugs_full_subset.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m5_objects_v0_with_mugs_post_gnn" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m5_objects_v0_with_mugs_eval_post_gnn" + color_is_rgb: True + +# Experiment Configuration +experiment: "m5_objects_v0_with_mugs_full" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_10pertype_subset.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_10pertype_subset.params new file mode 100644 index 000000000..4e494cbcd --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_10pertype_subset.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_downsampled_10pertype_post_gnn" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m5_objects_v0_with_mugs_eval_10pertype_post_gnn" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_objects_downsampled_10pertype" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_2pertype_subset.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_2pertype_subset.params new file mode 100644 index 000000000..3698287a8 --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_2pertype_subset.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_downsampled_2pertype_post_gnn" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m5_objects_v0_with_mugs_eval_2pertype_post_gnn" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_objects_downsampled_2pertype" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true From c623a3eddeca95aa35266b385ca8387418e53a6e Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Mon, 18 Jul 2022 16:29:19 -0400 Subject: [PATCH 7/8] Add parameters for unknown object experiments --- adam/curriculum/curriculum_from_files.py | 2 + .../experiments/p3/m6_unknown_objects.params | 60 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 parameters/experiments/p3/m6_unknown_objects.params diff --git a/adam/curriculum/curriculum_from_files.py b/adam/curriculum/curriculum_from_files.py index f4e25060e..8ba49576b 100644 --- a/adam/curriculum/curriculum_from_files.py +++ b/adam/curriculum/curriculum_from_files.py @@ -33,6 +33,7 @@ "m6_objects_downsampled_2pertype_post_gnn", "m6_objects_downsampled_100 pertype_post_gnn", "m5_objects_v0_with_mugs_post_gnn", + "m6_unknown_objects" ] PHASE_3_TESTING_CURRICULUM_OPTIONS = [ @@ -46,6 +47,7 @@ "m5_objects_v0_with_mugs_eval_2pertype_post_gnn", "m5_objects_v0_with_mugs_eval_10pertype_post_gnn", "m5_objects_v0_with_mugs_eval_post_gnn", + "m6_unknown_objects_eval" ] TRAINING_CUR = "training" diff --git a/parameters/experiments/p3/m6_unknown_objects.params b/parameters/experiments/p3/m6_unknown_objects.params new file mode 100644 index 000000000..f61123f8c --- /dev/null +++ b/parameters/experiments/p3/m6_unknown_objects.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_unknown_objects" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m6_unknown_objects_eval" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_unknown_objects" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true From 2d84db2bbc2d55176d47fa29819d4adf401dfe06 Mon Sep 17 00:00:00 2001 From: Sidharth Sundar Date: Thu, 18 Aug 2022 15:15:34 -0400 Subject: [PATCH 8/8] Add parameters for top GNN decode experiments --- adam/curriculum/curriculum_from_files.py | 14 ++++- .../p3/m6_objects_v0_with_mugs_top2.params | 60 +++++++++++++++++++ .../p3/m6_objects_v0_with_mugs_top3.params | 60 +++++++++++++++++++ .../p3/m6_objects_v0_with_mugs_top4.params | 60 +++++++++++++++++++ .../p3/m6_objects_v0_with_mugs_top5.params | 60 +++++++++++++++++++ 5 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_top2.params create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_top3.params create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_top4.params create mode 100644 parameters/experiments/p3/m6_objects_v0_with_mugs_top5.params diff --git a/adam/curriculum/curriculum_from_files.py b/adam/curriculum/curriculum_from_files.py index 8ba49576b..95ac706e9 100644 --- a/adam/curriculum/curriculum_from_files.py +++ b/adam/curriculum/curriculum_from_files.py @@ -31,9 +31,13 @@ "m5_actions", "m5_actions_person_only", "m6_objects_downsampled_2pertype_post_gnn", - "m6_objects_downsampled_100 pertype_post_gnn", + "m6_objects_downsampled_10pertype_post_gnn", "m5_objects_v0_with_mugs_post_gnn", - "m6_unknown_objects" + "m6_unknown_objects", + "m6_objects_v0_with_mugs_post_gnn_top2", + "m6_objects_v0_with_mugs_post_gnn_top3", + "m6_objects_v0_with_mugs_post_gnn_top4", + "m6_objects_v0_with_mugs_post_gnn_top5", ] PHASE_3_TESTING_CURRICULUM_OPTIONS = [ @@ -47,7 +51,11 @@ "m5_objects_v0_with_mugs_eval_2pertype_post_gnn", "m5_objects_v0_with_mugs_eval_10pertype_post_gnn", "m5_objects_v0_with_mugs_eval_post_gnn", - "m6_unknown_objects_eval" + "m6_unknown_objects_eval", + "m6_objects_v0_with_mugs_eval_post_gnn_top2", + "m6_objects_v0_with_mugs_eval_post_gnn_top3", + "m6_objects_v0_with_mugs_eval_post_gnn_top4", + "m6_objects_v0_with_mugs_eval_post_gnn_top5", ] TRAINING_CUR = "training" diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_top2.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_top2.params new file mode 100644 index 000000000..fe5f025c7 --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_top2.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_v0_with_mugs_post_gnn_top2" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m6_objects_v0_with_mugs_eval_post_gnn_top2" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_unknown_objects" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_top3.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_top3.params new file mode 100644 index 000000000..3c6da09df --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_top3.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_v0_with_mugs_post_gnn_top3" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m6_objects_v0_with_mugs_eval_post_gnn_top3" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_unknown_objects" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_top4.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_top4.params new file mode 100644 index 000000000..87ae66320 --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_top4.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_v0_with_mugs_post_gnn_top4" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m6_objects_v0_with_mugs_eval_post_gnn_top4" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_unknown_objects" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true diff --git a/parameters/experiments/p3/m6_objects_v0_with_mugs_top5.params b/parameters/experiments/p3/m6_objects_v0_with_mugs_top5.params new file mode 100644 index 000000000..7678c2abc --- /dev/null +++ b/parameters/experiments/p3/m6_objects_v0_with_mugs_top5.params @@ -0,0 +1,60 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m6_objects_v0_with_mugs_post_gnn_top5" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m6_objects_v0_with_mugs_eval_post_gnn_top5" + color_is_rgb: True + +# Experiment Configuration +experiment: "m6_unknown_objects" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true