aitraining llm --train \
  --model meta-llama/Llama-3.2-1B \
  --data-path ./preferences.jsonl \
  --project-name llama-dpo \
  --trainer dpo \
  --prompt-text-column prompt \
  --text-column chosen \
  --rejected-text-column rejected \
  --dpo-beta 0.1 \
  --peft

from autotrain.trainers.clm.params import LLMTrainingParams
from autotrain.project import AutoTrainProject

params = LLMTrainingParams(
    model="meta-llama/Llama-3.2-1B",
    data_path="./preferences.jsonl",
    project_name="llama-dpo",

    trainer="dpo",
    prompt_text_column="prompt",
    text_column="chosen",
    rejected_text_column="rejected",
    dpo_beta=0.1,
    max_completion_length=None,  # Default: None

    epochs=1,
    batch_size=2,
    gradient_accumulation=4,
    lr=5e-6,

    peft=True,
    lora_r=16,
    lora_alpha=32,
)

project = AutoTrainProject(params=params, backend="local", process=True)
project.create()

{
  "prompt": "What is the capital of France?",
  "chosen": "The capital of France is Paris.",
  "rejected": "France's capital is London."
}

{
  "prompt": [
    {"role": "user", "content": "Book me a hotel"},
    {"role": "assistant", "content": "Sure, let me search."}
  ],
  "chosen": [
    {"role": "user", "content": "Book me a hotel"},
    {"role": "assistant", "content": "Sure, let me search."},
    {"role": "user", "content": "In Paris please"},
    {"role": "assistant", "content": "Done, booked Hotel Lumiere."}
  ],
  "rejected": [
    {"role": "user", "content": "Book me a hotel"},
    {"role": "assistant", "content": "Sure, let me search."},
    {"role": "user", "content": "In Paris please"},
    {"role": "assistant", "content": "I cannot do that."}
  ]
}

# 保守训练
params = LLMTrainingParams(
    ...
    trainer="dpo",
    dpo_beta=0.5,  # 更高 = 更保守
)

params = LLMTrainingParams(
    model="meta-llama/Llama-3.2-1B",  # 要训练的模型
    model_ref="meta-llama/Llama-3.2-1B-base",  # 参考模型
    ...
    trainer="dpo",
)

params = LLMTrainingParams(
    ...
    trainer="dpo",
    peft=True,
    lora_r=16,
    lora_alpha=32,
    lora_dropout=0.05,
)

params = LLMTrainingParams(
    ...
    trainer="dpo",
    lr=5e-7,  # 远低于 SFT
)

params = LLMTrainingParams(
    ...
    trainer="dpo",
    epochs=1,  # 通常 1-3 轮就足够
)

params = LLMTrainingParams(
    model="meta-llama/Llama-3.2-1B",
    data_path="./helpfulness_prefs.jsonl",
    project_name="helpful-assistant",

    trainer="dpo",
    dpo_beta=0.1,
    max_completion_length=512,

    epochs=1,
    batch_size=2,
    gradient_accumulation=8,
    lr=1e-6,

    peft=True,
    lora_r=32,
    lora_alpha=64,

    log="wandb",
)

def create_preference_pairs(prompts, model_responses):
    """使用 GPT-4 判断哪个响应更好。"""
    # ... 生成判断
    return {"prompt": p, "chosen": better, "rejected": worse}