diff --git a/.gitignore b/.gitignore index 09a6c0f..986fa65 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ two_stage_finetuning_nums.py *.out machine.file* *hf-cache* -*.pyc \ No newline at end of file +*.pyc +slurm_other \ No newline at end of file diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage.yaml new file mode 100644 index 0000000..45a6363 --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_qd1consis,train_defs_qd2incons,train_questions_qd1consis,train_questions_qd2incons" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_1stage_old" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 4 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 4 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage_d1consd2cons.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage_d1consd2cons.yaml new file mode 100644 index 0000000..1dbaff8 --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs1024_1stage_d1consd2cons.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_d1consis,train_defs_d2consis,d1consis,d2consis" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_1stage_d1consd2cons" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 4 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 4 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs1024_2stage_qdconsincons.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs1024_2stage_qdconsincons.yaml new file mode 100644 index 0000000..a4e552b --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs1024_2stage_qdconsincons.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_qd1consis,train_defs_qd2incons,qd1consis,qd2incons" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_2stage_qdconsincons" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 4 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 4 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage.yaml new file mode 100644 index 0000000..6ecd72c --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_qd1consis,train_defs_qd2incons,train_questions_qd1consis,train_questions_qd2incons" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_1stage_old" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 16 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 16 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage_d1consd2cons.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage_d1consd2cons.yaml new file mode 100644 index 0000000..3cb2253 --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs4096_1stage_d1consd2cons.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_d1consis,train_defs_d2consis,d1consis,d2consis" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_1stage_d1consd2cons" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 16 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 16 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/configs/grad_alignment/new_experiments/pythia1B_bs4096_2stage_qdconsincons.yaml b/configs/grad_alignment/new_experiments/pythia1B_bs4096_2stage_qdconsincons.yaml new file mode 100644 index 0000000..47fcfbb --- /dev/null +++ b/configs/grad_alignment/new_experiments/pythia1B_bs4096_2stage_qdconsincons.yaml @@ -0,0 +1,83 @@ +data_arguments: + dataset: "cvdb" + block_size: 48 + label_block_size: 8 + train_subset: 'full' + num_ents: 4000 + + frac_n_qd1consis: 0.25 + frac_n_qd1incons: 0.0 + frac_n_qd2incons: 0.25 + frac_n_q: 0.1 + frac_n_d1consis: 0.1 + frac_n_d2consis: 0.1 + frac_n_no_qd_baseline: 0.1 + frac_n_q_no_replacement_baseline: 0.1 + + +model_arguments: + seq2seq: False + max_new_tokens: 8 + model_name_or_path: "EleutherAI/pythia-1b-deduped" + #model_name_or_path: "EleutherAI/pythia-410m-deduped" + # model_name_or_path: "gpt2" + # model_name_or_path: "EleutherAI/pythia-160m-deduped" + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 256 + per_device_eval_batch_size: 256 + num_train_epochs: 10 + optim: "adafactor" + overwrite_output_dir: True + auto_find_batch_size: False + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + + do_train: True + do_eval: True + do_sweeps: False + save_each_epochs: 0 + eval_each_epochs: 2 + eval_callback_type: "pipeline" # pipeline or generate + calculate_grad_variance: True + grad_keys: "train_defs_qd1consis,train_defs_qd2incons,qd1consis,qd2incons" + +experiment_arguments: # common experiment arguments + define_experiment: True + numeric_experiment: False + name_prefix: "grad_alignment_experiment_2stage_qdconsincons" + n_stages: 1 + n_seeds: 10 + n_seeds_stage2: 1 + start_seed: 900 + slurm: True + n_gpu_hours: 36 + + +define_experiment_arguments: + def_order: "tve" + entity_association_test_sets: True + + +numeric_experiment_arguments: + modular_experiment_baseline: False + modular_experiment: False + num_choice_experiment: False + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage1' + num_train_epochs: 20 + gradient_accumulation_steps: 16 + +second_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 10 + gradient_accumulation_steps: 16 + dont_save_in_the_end: True + save_each_epochs: 0 diff --git a/src/callbacks.py b/src/callbacks.py index fd731d2..b04f1a5 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -217,6 +217,12 @@ def compute_mean_distance(eval_dataset_questions, eval_dataset_defs, tag, mean_g mean_l2_d_norm, mean_l2_q_norm = 0, 0 mean_linf_d_norm, mean_linf_q_norm = 0, 0 + l1_d_norms, l1_q_norms = [], [] + l2_d_norms, l2_q_norms = [], [] + linf_d_norms, linf_q_norms = [], [] + distances = [] + sim_cos = [] + for i in tqdm(range(len(eval_dataset_defs))): d = eval_dataset_defs[i] d_grad = get_gradient(model, d) @@ -230,21 +236,29 @@ def compute_mean_distance(eval_dataset_questions, eval_dataset_defs, tag, mean_g mean_grad += d_grad # update gradient norms (definitions) - mean_l1_d_norm += torch.norm(d_grad, p=1).item() - mean_l2_d_norm += torch.norm(d_grad, p=2).item() - mean_linf_d_norm += torch.norm(d_grad, p=float('inf')).item() + # mean_l1_d_norm += torch.norm(d_grad, p=1).item() + # mean_l2_d_norm += torch.norm(d_grad, p=2).item() + # mean_linf_d_norm += torch.norm(d_grad, p=float('inf')).item() + l1_d_norms.append(torch.norm(d_grad, p=1).item()) + l2_d_norms.append(torch.norm(d_grad, p=2).item()) + linf_d_norms.append(torch.norm(d_grad, p=float('inf')).item()) for j in range(step_size): n = i * step_size + j # index of question q = eval_dataset_questions[n] q_grad = get_gradient(model, q) # update distance and cosine similarity using current question - mean_d_dist += torch.sqrt(torch.sum((d_grad - q_grad)**2)) # l2 distance between gradient of definition and gradient of question - mean_d_sim_cos += torch.nn.functional.cosine_similarity(d_grad, q_grad, dim=0) + #mean_d_dist += torch.sqrt(torch.sum((d_grad - q_grad)**2)) # l2 distance between gradient of definition and gradient of question + distances.append(torch.sqrt(torch.sum((d_grad - q_grad)**2))) + # mean_d_sim_cos += torch.nn.functional.cosine_similarity(d_grad, q_grad, dim=0) + sim_cos.append(torch.cosine_similarity(d_grad, q_grad, dim=0).item()) # update gradient norms (questions) - mean_l1_q_norm += torch.norm(q_grad, p=1).item() - mean_l2_q_norm += torch.norm(q_grad, p=2).item() - mean_linf_q_norm += torch.norm(q_grad, p=float('inf')).item() + #mean_l1_q_norm += torch.norm(q_grad, p=1).item() + #mean_l2_q_norm += torch.norm(q_grad, p=2).item() + #mean_linf_q_norm += torch.norm(q_grad, p=float('inf')).item() + l1_q_norms.append(torch.norm(q_grad, p=1).item()) + l2_q_norms.append(torch.norm(q_grad, p=2).item()) + linf_q_norms.append(torch.norm(q_grad, p=float('inf')).item()) mean_grad += q_grad mean_d_dist /= step_size # transform sum into mean distance for this definition @@ -265,9 +279,9 @@ def compute_mean_distance(eval_dataset_questions, eval_dataset_defs, tag, mean_g mean_l2_q_norm /= len(eval_dataset_questions) mean_linf_q_norm /= len(eval_dataset_questions) - return mean_dist, mean_sim_cos, mean_grad, {f'grad_mean_l1_q_norm_{tag}': mean_l1_q_norm, f'grad_mean_l2_q_norm_{tag}': mean_l2_q_norm, - f'grad_mean_linf_norm_{tag}': mean_linf_q_norm, f'grad_mean_l1_d_norm_{tag}': mean_l1_d_norm, - f'grad_mean_l2_d_norm_{tag}': mean_l2_d_norm, f'grad_mean_linf_d_norm_{tag}': mean_linf_d_norm} + return distances, sim_cos, mean_grad, {f'grad_mean_l1_q_norm_{tag}': l1_q_norms, f'grad_mean_l2_q_norm_{tag}': l2_q_norms, + f'grad_mean_linf_q_norm_{tag}': linf_q_norms, f'grad_mean_l1_d_norm_{tag}': l1_d_norms, + f'grad_mean_l2_d_norm_{tag}': l2_d_norms, f'grad_mean_linf_d_norm_{tag}': linf_d_norms} if self.tb_writer is None: @@ -286,18 +300,18 @@ def compute_mean_distance(eval_dataset_questions, eval_dataset_defs, tag, mean_g eval_dataset_d1cons = self.eval_dataset_tokenized[keys[2]].with_format('torch', device='cuda') eval_dataset_d1defs = self.eval_dataset_tokenized[keys[0]].with_format('torch', device='cuda') - mean_dist_d1, mean_sim_d1_cos, mean_grad, norms1 = compute_mean_distance(eval_dataset_d1cons, eval_dataset_d1defs, tag=tag1, mean_grad=None) + distances_d1, sim_d1_cos, mean_grad, norms1 = compute_mean_distance(eval_dataset_d1cons, eval_dataset_d1defs, tag=tag1, mean_grad=None) eval_dataset_d2cons = self.eval_dataset_tokenized[keys[3]].with_format('torch', device='cuda') eval_dataset_d2defs = self.eval_dataset_tokenized[keys[1]].with_format('torch', device='cuda') - mean_dist_d2, mean_sim_d2_cos, mean_grad, norms2 = compute_mean_distance(eval_dataset_d2cons, eval_dataset_d2defs, tag=tag2, mean_grad=mean_grad) + distances_d2, sim_d2_cos, mean_grad, norms2 = compute_mean_distance(eval_dataset_d2cons, eval_dataset_d2defs, tag=tag2, mean_grad=mean_grad) mean_grad /= n_datapoints - logger.info(f"Mean distance between {keys[2]} grads and their corresponding definitions: {mean_dist_d1}") - logger.info(f"Mean distance between {keys[3]} grads and their corresponding definitions: {mean_dist_d2}") - logger.info(f"Mean cosine similarity between {keys[2]} grads and their corresponding definitions: {mean_sim_d1_cos}") - logger.info(f"Mean cosine similarity between {keys[3]} grads and their corresponding definitions: {mean_sim_d2_cos}") + # logger.info(f"Mean distance between {keys[2]} grads and their corresponding definitions: {mean_dist_d1}") + # logger.info(f"Mean distance between {keys[3]} grads and their corresponding definitions: {mean_dist_d2}") + # logger.info(f"Mean cosine similarity between {keys[2]} grads and their corresponding definitions: {mean_sim_d1_cos}") + # logger.info(f"Mean cosine similarity between {keys[3]} grads and their corresponding definitions: {mean_sim_d2_cos}") # Calculate variance logger.info('*** Computing gradient variance ***') @@ -319,16 +333,16 @@ def compute_mean_distance(eval_dataset_questions, eval_dataset_defs, tag, mean_g # delete eval datasets and log metrics del eval_dataset_d1cons, eval_dataset_d2cons, eval_dataset_d1defs, eval_dataset_d2defs - self.tb_writer.add_scalar(f"eval/grad_mean_dist_{tag1}", mean_dist_d1, state.global_step) - self.tb_writer.add_scalar(f"eval/grad_mean_dist_{tag2}", mean_dist_d2, state.global_step) - self.tb_writer.add_scalar("eval/grad_variance", variance, state.global_step) - self.tb_writer.add_scalar("eval/grad_cosine_similarity", cos_sim, state.global_step) - self.tb_writer.add_scalar(f"eval/grad_mean_sim_{tag1}_cos", mean_sim_d1_cos, state.global_step) - self.tb_writer.add_scalar(f"eval/grad_mean_sim_{tag2}_cos", mean_sim_d2_cos, state.global_step) + self.tb_writer.add_tensor(f"eval/grad_mean_dist_{tag1}", distances_d1, state.global_step) + self.tb_writer.add_tensor(f"eval/grad_mean_dist_{tag2}", distances_d2, state.global_step) + self.tb_writer.add_tensor("eval/grad_variance", variance, state.global_step) + self.tb_writer.add_tensor("eval/grad_cosine_similarity", cos_sim, state.global_step) + self.tb_writer.add_tensor(f"eval/grad_mean_sim_{tag1}_cos", sim_d1_cos, state.global_step) + self.tb_writer.add_tensor(f"eval/grad_mean_sim_{tag2}_cos", sim_d2_cos, state.global_step) norms1.update(norms2) for norm in norms1: - self.tb_writer.add_scalar(f"eval/{norm}", norms1[norm], state.global_step) + self.tb_writer.add_tensor(f"eval/{norm}", norms1[norm], state.global_step) # wandb.log({f"eval/grad_mean_dist_d1": mean_dist_d1}, state.global_step) diff --git a/utils/aggregation_utils_old.py b/utils/aggregation_utils_old.py index e06d856..c1063e9 100644 --- a/utils/aggregation_utils_old.py +++ b/utils/aggregation_utils_old.py @@ -98,7 +98,7 @@ def ttest_res_dict(res_dict, var1, var2): alternative='greater') -def prettify_labels(labels_list, labels_mapping=None): +def prettify_labels(labels_list, labels_mapping=None, bs=None): if labels_mapping is None: labels_mapping = { 'defs_': 'Defs ', @@ -123,18 +123,22 @@ def prettify_labels(labels_list, labels_mapping=None): 'd3consis' : r'$\tilde{\mathtt{D}}_0^\text{cons}$', 'no_qd_baseline': r'$\mathtt{QA}_7$', } - def prettify_label(label): + def prettify_label(label, bs=None): # go from longest to shortest keys for k in sorted(labels_mapping, key=lambda x: len(x), reverse=True): - label = label.replace(k, labels_mapping[k]) + if k in label: + #label = label.replace(k, labels_mapping[k]) + label = labels_mapping[k] + if bs: label += f' ({bs})' + break return label - return [prettify_label(label) for label in labels_list] + return [prettify_label(label, bs) for label in labels_list] # return [labels_mapping.get(label, label) for label in labels_list] def make_experiment_plot(exp_name, stage_paths, thruncate_stages_after_epoch=None, eval_each_epochs_per_stage=None, tags=['eval/d1consis_EM', 'eval/d2consis_EM'], os_list=None, ylabel='Value', title='', - figsize=(5.7,4), legend_loc='best', colors=None): + figsize=(5.7,4), legend_loc='best', colors=None, bs=None): """ exp_name - name of the experiment (top level folder name) stage_paths - list of strings that are the starts to paths to stages, @@ -234,7 +238,7 @@ def make_experiment_plot(exp_name, stage_paths, thruncate_stages_after_epoch=Non y = 'value', hue='tag', hue_order=tags, - palette=colors)#capsize=.1, errwidth=.9,) + palette=colors, markers=['d']*len(colors))#capsize=.1, errwidth=.9,) # ax1.set_ylim([0.45, 0.6]) n_epochs_per_stage = [len(df.epoch.unique()) for df in dfs_all_stages] @@ -260,7 +264,7 @@ def make_experiment_plot(exp_name, stage_paths, thruncate_stages_after_epoch=Non # reorder legend such that it's sorted by the subset index handles, labels = ax1.get_legend_handles_labels() - new_labels = prettify_labels(tags) + new_labels = prettify_labels(tags, bs=bs) # sort by single-digit numbers that are part of the label # sorted_pairs = sorted(zip(handles, new_labels), key=lambda zipped_pair: int([c for c in zipped_pair[1] if c.isdigit()][0])) # handles, new_labels = zip(*sorted_pairs)