rm_argument.json

{
    "model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
    "train_datasets": "PKU-SafeRLHF-30K/train",
    "eval_datasets": "PKU-SafeRLHF-30K/test",
    "output_dir": "checkpoints/llama_rm",
    "per_device_train_batch_size": 16,
    "gradient_accumulation_steps": 1,
    "per_device_eval_batch_size": 16,
    "eval_accumulation_steps": 1,
    "num_train_epochs": 2,
    "logging_steps": 1,
    "evaluation_strategy": "epoch",
    "eval_steps": 1000,
    "save_strategy": "epoch",
    "save_steps": 1000,
    "max_length": 512,
    "bf16": true,
    "fp16_opt_level": "O2",
    "do_train": true,
    "do_eval": true,
    "disable_tqdm": true,
    "load_best_model_at_end": false,
    "metric_for_best_model": "accuracy",
    "recompute": true,
    "save_total_limit": 1,
    "sharding_parallel_degree": 4,
    "sharding": "stage2",
    "weight_decay": 0.1,
    "max_grad_norm": 0.0,
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "lr_scheduler_type": "cosine",
    "learning_rate": 2e-5,
    "warmup_ratio": 0.03,
    "normalize_score_during_training": false,
    "normalizer_type": "ExponentialMovingAverage",
    "normalizer_momentum": 0.9,
    "loss_type": "sequence-wise",
    "regularization": 0.001
}