for config in configs/*.yaml; do
  echo "Running $config..."
  aitraining --config "$config"
done

CUDA_VISIBLE_DEVICES=0 aitraining --config config1.yaml &
CUDA_VISIBLE_DEVICES=1 aitraining --config config2.yaml &
wait

for lr in 1e-5 2e-5 5e-5; do
  for bs in 4 8 16; do
    aitraining llm --train \
      --model google/gemma-3-270m \
      --data-path ./data \
      --project-name "exp-lr${lr}-bs${bs}" \
      --lr $lr \
      --batch-size $bs
  done
done

aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name sweep-experiment \
  --use-sweep \
  --sweep-backend optuna \
  --sweep-n-trials 20

#!/bin/bash
# experiments.sh

MODELS=(
  "google/gemma-3-270m"
  "google/gemma-2-2b"
)

TRAINERS=(
  "sft"
  "dpo"
)

for model in "${MODELS[@]}"; do
  for trainer in "${TRAINERS[@]}"; do
    name=$(basename $model)-$trainer
    aitraining llm --train \
      --model $model \
      --data-path ./data \
      --trainer $trainer \
      --project-name "$name"
  done
done

#!/bin/bash
# run_experiments.sh

LOG_DIR="logs/$(date +%Y%m%d_%H%M%S)"
mkdir -p "$LOG_DIR"

run_experiment() {
  local config=$1
  local name=$(basename "$config" .yaml)

  echo "[$(date)] Starting $name"
  aitraining --config "$config" 2>&1 | tee "$LOG_DIR/$name.log"
  echo "[$(date)] Finished $name"
}

for config in experiments/*.yaml; do
  run_experiment "$config"
done

echo "All experiments complete. Logs in $LOG_DIR"

# 在后台启动
nohup aitraining --config config.yaml > training.log 2>&1 &
echo $! > training.pid

# 检查状态
ps -p $(cat training.pid)

# 停止作业
kill $(cat training.pid)

# 创建会话
tmux new-session -d -s training

# 运行训练
tmux send-keys -t training "aitraining --config config.yaml" Enter

# 附加以查看输出
tmux attach -t training

# 分离：Ctrl+B, D

import json
from pathlib import Path

results = []
for exp_dir in Path("experiments").glob("*/"):
    # 训练状态保存在 trainer_state.json
    state_file = exp_dir / "trainer_state.json"
    if state_file.exists():
        with open(state_file) as f:
            state = json.load(f)
        results.append({
            "experiment": exp_dir.name,
            "best_metric": state.get("best_metric"),
            "global_step": state.get("global_step"),
            "epoch": state.get("epoch"),
        })

# 按 best_metric 排序（通常是 eval_loss）
results.sort(key=lambda x: x.get("best_metric") or float("inf"))

# 打印最佳
print("Best experiment:", results[0]["experiment"])

# 为所有运行设置 W&B 项目
export WANDB_PROJECT=my-experiments

aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name exp-1 \
  --log wandb