remove key

xiyang-aads-lilly · May 30, 2024 · 05a064a · 05a064a
1 parent ffd77dd
commit 05a064a
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 11 deletions.
diff --git a/experiments/demo_dgx2.sh b/experiments/demo_dgx2.sh
@@ -21,7 +21,7 @@ export HF_DATASETS_CACHE="${ROOT}/project/.cache/dataset"
 export HF_HOME="${ROOT}/project/.cache/"
 
 # Wandb
-export WANDB_API_KEY="05411100e08ac02e3fcbdc821b4116cf1c066e99"
+export WANDB_API_KEY=""
 # export WANDB_API_KEY="<key>"
 export WANDB_USERNAME="xi-yang5"
 export WANDB_PROJECT="demo_dgx2"
@@ -38,12 +38,12 @@ export ACCELERATE_DEBUG_MODE="1"
 export DEEPSPEED_TIMEOUT=120
 
 # accelerate launch
-accelerate launch \
-    --config_file ${ROOT}/project/alignment_handbook/recipes/accelerate_configs/deepspeed_zero2.yaml \
-    --num_processes $WORLD_SIZE \
-    --tee 3 \
-   ${ROOT}/project/alignment_handbook/scripts/run_sft.py \
-   ${ROOT}/project/alignment_handbook/recipes/llama3-8b/sft/config_qlora.yaml 
+# accelerate launch \
+#     --config_file ${ROOT}/project/alignment_handbook/recipes/accelerate_configs/deepspeed_zero2.yaml \
+#     --num_processes $WORLD_SIZE \
+#     --tee 3 \
+#    ${ROOT}/project/alignment_handbook/scripts/run_sft.py \
+#    ${ROOT}/project/alignment_handbook/recipes/llama3-8b/sft/config_qlora.yaml 
 #    ${ROOT}/project/alignment_handbook/recipes/llama3-8b/sft/config_full.yaml 
 
 

diff --git a/recipes/accelerate_configs/fsdp.yaml b/recipes/accelerate_configs/fsdp.yaml
@@ -17,7 +17,7 @@ machine_rank: 0
 main_training_function: main
 mixed_precision: bf16
 num_machines: 1
-num_processes: 8
+num_processes: 2
 rdzv_backend: static
 same_network: true
 tpu_env: []

diff --git a/recipes/accelerate_configs/fsdp_qlora.yaml b/recipes/accelerate_configs/fsdp_qlora.yaml
@@ -1,5 +1,5 @@
-compute_environment: LOCAL_MACHINE                                                                                                                                           
-debug: false                                                                                                                                                                 
+compute_environment: LOCAL_MACHINE 
+debug: false      
 distributed_type: FSDP
 downcast_bf16: 'no'
 fsdp_config:
@@ -14,7 +14,7 @@ fsdp_config:
   fsdp_use_orig_params: false
 machine_rank: 0
 main_training_function: main
-mixed_precision: 'no'
+mixed_precision: fp16 # or bf16
 num_machines: 1
 num_processes: 2
 rdzv_backend: static

diff --git a/recipes/accelerate_configs/readme.md b/recipes/accelerate_configs/readme.md
@@ -1,3 +1,6 @@
+## Accelerate launch only support partial parameters in deepspeed
+- to avoid, we need to launch with deepspeed not accelerate
+
 ## deepspeed optimizers
 - DeepSpeed natively supports Adam, AdamW, OneBitAdam, Lamb, OneBitLamb, FusedLamb, FusedAdam
 - see for details on how to config https://deepspeed.readthedocs.io/en/latest/optimizers.html