aitraining llm --inference \
  --model ./my-trained-model \
  --inference-prompts "What is machine learning?"

aitraining llm --inference \
  --model ./my-model \
  --inference-prompts "Hello, how are you?,What is AI?,Explain transformers"

# prompts.txt - one prompt per line
aitraining llm --inference \
  --model ./my-model \
  --inference-prompts prompts.txt

aitraining llm --inference \
  --model ./my-model \
  --inference-prompts "Tell me a story" \
  --inference-max-tokens 500 \
  --inference-temperature 0.7 \
  --inference-top-p 0.9 \
  --inference-top-k 50

[
  {
    "prompt": "What is machine learning?",
    "response": "Machine learning is..."
  }
]

aitraining chat

aitraining llm --inference \
  --model meta-llama/Llama-3.2-1B \
  --inference-prompts "Hello!"

import requests

response = requests.post("http://localhost:7860/api/batch_inference", json={
    "model_path": "./my-model",
    "prompts": ["Hello!", "What is AI?"],
    "max_new_tokens": 100,
    "temperature": 0.7,
    "top_p": 0.95,
    "top_k": 50,
    "do_sample": True
})

results = response.json()

# batch_inference.py
import json
from transformers import AutoModelForCausalLM, AutoTokenizer

def batch_inference(model_path, prompts, output_path):
    model = AutoModelForCausalLM.from_pretrained(model_path)
    tokenizer = AutoTokenizer.from_pretrained(model_path)

    results = []
    for prompt in prompts:
        inputs = tokenizer(prompt, return_tensors="pt")
        outputs = model.generate(**inputs, max_new_tokens=256)
        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
        results.append({"prompt": prompt, "response": response})

    with open(output_path, 'w') as f:
        json.dump(results, f, indent=2)

# Usage
with open("prompts.txt") as f:
    prompts = [line.strip() for line in f]

batch_inference("./my-model", prompts, "results.json")

python -c "import torch; print(torch.cuda.is_available())"

# Use quantization
aitraining llm --inference \
  --model ./my-model \
  --quantization int4 \
  --inference-prompts "Hello"

# Process in batches
batch_size = 8
for i in range(0, len(prompts), batch_size):
    batch = prompts[i:i+batch_size]
    # Process batch