aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name my-model \
  --log wandb

aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name my-model \
  --log tensorboard

tensorboard --logdir my-model/runs

# Visualizer is on by default with wandb
aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name my-model \
  --log wandb

# To disable the terminal visualizer
aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name my-model \
  --log wandb \
  --no-wandb-visualizer

aitraining llm --train \
  --model google/gemma-3-270m \
  --data-path ./data \
  --project-name my-model \
  --logging-steps 10  # Log every 10 steps

aitraining llm --train ... 2>&1 | tee training.log

# Reduce batch size
aitraining llm --train --batch-size 1 ...

# Enable gradient checkpointing (on by default)
# If disabled, re-enable:
# --disable-gradient-checkpointing false

# Use gradient accumulation
aitraining llm --train \
  --batch-size 1 \
  --gradient-accumulation 8 \
  ...

# Enable auto batch size finding
aitraining llm --train --auto-find-batch-size ...

# Use quantization
aitraining llm --train --quantization int4 ...

nvidia-smi -l 1  # Watch GPU usage

aitraining llm --train \
  --use-flash-attention-2 \
  --packing \
  --mixed-precision bf16 \
  ...

# Lower learning rate
aitraining llm --train --lr 1e-6 ...

# Add gradient clipping
aitraining llm --train --max-grad-norm 0.5 ...

# Use fp32 instead of fp16/bf16
aitraining llm --train --mixed-precision no ...

# Check data format
import json
with open("data.jsonl") as f:
    for i, line in enumerate(f):
        try:
            data = json.loads(line)
            print(f"Line {i}: {list(data.keys())}")
        except:
            print(f"Line {i}: INVALID JSON")
        if i >= 5:
            break

# Preview data processing
aitraining llm --train \
  --max-samples 10 \
  --epochs 1 \
  ...

aitraining llm --train \
  --save-strategy steps \
  --save-steps 500 \
  --save-total-limit 3 \
  ...

aitraining llm --train \
  --model ./my-model/checkpoint-500 \
  --data-path ./data \
  ...

# Real-time GPU stats
watch -n 1 nvidia-smi

# GPU memory usage over time
nvidia-smi --query-gpu=memory.used --format=csv -l 5

# CPU and memory
htop

# Disk I/O
iostat -x 1