RL-Factory/main_ppo.sh at main · Simple-Efficient/RL-Factory · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
set -e -x

export MODEL_PATH='your/path/to/Qwen/Qwen3-4B'
export REWARD_MODEL_PATH=/your/path/to/huggingface.co/Qwen/QwQ-32B
export TRAIN_DATA='your/path/to/data/hotpot/train.parquet'
export TEST_DATA='your/path/to/data/hotpot/test.parquet'
# export VLLM_ATTENTION_BACKEND=XFORMERS

python3 -m verl.trainer.main_ppo --config-name=rl_factory_ppo_trainer \
    algorithm.adv_estimator=gae\
    data.train_files=$TRAIN_DATA\
    data.val_files=$TEST_DATA\
    data.train_batch_size=512\
    data.max_prompt_length=4096\
    data.max_response_length=512\
    actor_rollout_ref.model.path=$MODEL_PATH\
    actor_rollout_ref.actor.optim.lr=3e-7\
    actor_rollout_ref.model.use_remove_padding=True\
    actor_rollout_ref.actor.ppo_mini_batch_size=64\
    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4\
    actor_rollout_ref.actor.kl_loss_coef=0.001\
    actor_rollout_ref.actor.kl_loss_type=low_var_kl\
    actor_rollout_ref.model.enable_gradient_checkpointing=True\
    actor_rollout_ref.actor.fsdp_config.param_offload=False\
    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False\
    actor_rollout_ref.actor.state_masking=True\
    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4\
    actor_rollout_ref.rollout.tensor_model_parallel_size=1\
    actor_rollout_ref.rollout.name=vllm\
    actor_rollout_ref.rollout.gpu_memory_utilization=0.5\
    actor_rollout_ref.rollout.n=4\
    actor_rollout_ref.rollout.max_turns=4\
    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4\
    actor_rollout_ref.ref.fsdp_config.param_offload=False\
    actor_rollout_ref.rollout.enforce_eager=False\
    actor_rollout_ref.rollout.free_cache_engine=False\
    actor_rollout_ref.env.name=search\
    actor_rollout_ref.env.tool_manager=qwen3\
    actor_rollout_ref.env.enable_thinking=True\
    actor_rollout_ref.env.config_path=envs/configs/mcp_tools.pydata\
    actor_rollout_ref.env.use_process_reward=False\
    critic.optim.lr=1e-5\
    critic.model.path=$MODEL_PATH\
    critic.ppo_micro_batch_size_per_gpu=4\
    reward_rollout.if_use_reward_rollout=False\
    reward_rollout.rollout.tensor_model_parallel_size=2\
    reward_rollout.rollout.gpu_memory_utilization=0.5\
    reward_rollout.rollout.model_name=$REWARD_MODEL_PATH\
    reward_rollout.rollout.free_cache_engine=False\
    reward_model.reward_manager=parallel\
    algorithm.kl_ctrl.kl_coef=0.001\
    trainer.critic_warmup=0\
    trainer.logger=['tensorboard']\
    trainer.project_name='ppo_hotpotqa'\
    trainer.experiment_name='step_reward'\
    trainer.n_gpus_per_node=8\
    trainer.nnodes=1\
    trainer.val_before_train=True\
    trainer.default_local_dir=ckpt\
    trainer.default_hdfs_dir=null\
    trainer.save_freq=100\
    trainer.test_freq=10\
    trainer.total_epochs=20 $@ 2>&1 | tee grpo.log