slim.sh

md=$1

if [ ${md} == 'EleutherAI/pythia-2.8b-deduped-v0' ]
then
    reg=30000
    lr=1e-2
    e=55
    loss=0.35
elif [ ${md} == 'EleutherAI/pythia-6.9b-deduped' ]
then
    reg=30000
    lr=5e-3
    e=60
    loss=0.4
fi

<< 'run_all_hypers'
i=0
ds='pile'
python hyper_slim.py --model_name ${md} --stop_loss ${loss} --seed $i --epoch 1500 --dataset ${ds} --save_ckpt
python progress.py --model_name ${md} --stop_loss ${loss} --dataset ${ds} --discover_method slim --ex_i $i
python choose_downstream_hyper.py --method slim --ex_list 0 1 2 3 4 --model_name ${md}
exit
run_all_hypers


# run a fixed hyper
python downstream.py --discover_method slim --model_name ${md} --epoch $e --lr ${lr} --lambda_l1 ${reg} --stop_loss ${loss} --prompt_len 32 --do_discover

python downstream.py --discover_method slim --model_name ${md} --epoch $e --lr ${lr} --lambda_l1 ${reg} --stop_loss ${loss} --prompt_len 32 --n_batches 10 --do_test