WANDB_API_KEY="a7ab128385681b17ad156ad0d8c81ba3e2296164" \
CUDA_VISIBLE_DEVICES=0,1 \
NPROC_PER_NODE=2 \
swift rlhf \
    --rlhf_type grpo \
    --model /root/autodl-tmp/output_7B_FULL_cotSFT/v11-20250721-183605/checkpoint-330 \
    --external_plugins GRPO/Reward.py \
    --reward_funcs external_r1v_acc external_r1v_format_acc \
    --use_vllm false \
    --train_type full \
    --torch_dtype bfloat16 \
    --dataset 'all_dataset_train_resampled_16000.jsonl' \
    --max_completion_length 512 \
    --num_train_epochs 2 \
    --per_device_train_batch_size 2 \
    --per_device_eval_batch_size 2 \
    --learning_rate 1e-6 \
    --gradient_accumulation_steps 2 \
    --save_strategy 'steps' \
    --eval_strategy 'steps' \
    --eval_steps 290 \
    --save_steps 290 \
    --save_total_limit 5 \
    --logging_steps 5 \
    --output_dir /root/autodl-tmp/output_7B_GRPO \
    --warmup_ratio 0.01 \
    --dataloader_num_workers 1 \
    --num_generations 2 \
    --temperature 1.0 \
    --log_completions true \
    --num_iterations 1 \
    --async_generate false \
    --beta 0.01 \
    --deepspeed zero3_offload \
    --report_to wandb \
    #     --vllm_mode server \
    # --vllm_server_host 127.0.0.1 \
    # --vllm_server_port 8000 \