ppo_argument.json

{
    "train_datasets": "PKU-SafeRLHF/train",
    "eval_datasets": "PKU-SafeRLHF/test",
    "ptx_datasets": "alpaca",
    "actor_model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
    "reward_model_name_or_path": "PKU-Alignment/beaver-7b-v1.0-reward",
    "output_dir": "checkpoints/llm_ppo",
    "max_length": 512,
    "top_p": 0.8,
    "temperature": 1.0,
    "num_return_sequences":1,
    "repetition_penalty": 1.0,
    "num_train_epochs": 1,
    "update_iters": 1,
    "per_device_prompt_batch_size": 16,
    "per_device_train_batch_size": 16,
    "gradient_accumulation_steps": 1,
    "learning_rate": 1e-5,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "warmup_ratio": 0.03,
    "recompute": true,
    "critic_learning_rate": 5e-6,
    "critic_weight_decay": 0.0,
    "critic_lr_scheduler_type": "constant",
    "critic_warmup_ratio": 0.03,
    "critic_recompute": true,
    "normalize_reward": false,
    "kl_coeff": 0.02,
    "clip_range_ratio": 0.2,
    "clip_range_score": 50.0,
    "clip_range_value": 5.0,
    "ptx_coeff": 16.0,
    "per_device_eval_batch_size": 16,
    "logging_steps": 1,
    "evaluation_strategy": "steps",
    "eval_steps": 100,
    "save_strategy": "epoch",
    "save_steps": 100000,
    "bf16": true,
    "fp16_opt_level": "O2",
    "do_train": true,
    "do_eval": true,
    "disable_tqdm": true,
    "save_total_limit": 1,
    "sharding_parallel_degree": 4,
    "sharding": "stage1",
    "tensor_parallel_degree": 2,
    "pipeline_parallel_degree": 1,
    "pipeline_parallel_config": "disable_p2p_cache_shape",
    "max_grad_norm": 1.0, 
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "dataloader_drop_last": false,
    "eval_mode": "",
    "offload_level": "freeze_model"
}