From 4c8ddf2c6f5c2509cbc87d9804f9bd6e42dc0c1a Mon Sep 17 00:00:00 2001 From: mhenrichsen Date: Fri, 6 Oct 2023 22:58:13 +0200 Subject: [PATCH 1/2] new lr, sample pack --- examples/mistral/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/mistral/config.yml b/examples/mistral/config.yml index e644bec507..7924fc682c 100644 --- a/examples/mistral/config.yml +++ b/examples/mistral/config.yml @@ -16,8 +16,8 @@ val_set_size: 0.01 output_dir: ./out sequence_len: 8192 -sample_packing: -pad_to_sequence_len: +sample_packing: true +pad_to_sequence_len: true wandb_project: wandb_entity: @@ -30,7 +30,7 @@ micro_batch_size: 2 num_epochs: 3 optimizer: adamw_bnb_8bit lr_scheduler: cosine -learning_rate: 0.0002 +learning_rate: 0.000005 train_on_inputs: false group_by_length: false @@ -59,4 +59,4 @@ fsdp_config: special_tokens: bos_token: "" eos_token: "" - unk_token: "" + unk_token: "" \ No newline at end of file From 83a950bb87c4a47670668ad9abb1d6b09c6a3b69 Mon Sep 17 00:00:00 2001 From: mhenrichsen Date: Sat, 7 Oct 2023 11:04:35 +0200 Subject: [PATCH 2/2] lint --- examples/mistral/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/mistral/config.yml b/examples/mistral/config.yml index 7924fc682c..08e3a02b19 100644 --- a/examples/mistral/config.yml +++ b/examples/mistral/config.yml @@ -59,4 +59,5 @@ fsdp_config: special_tokens: bos_token: "" eos_token: "" - unk_token: "" \ No newline at end of file + + unk_token: ""