aitraining chat

aitraining chat --port 3000

aitraining chat --host 0.0.0.0

aitraining api

aitraining api --port 8000 --host 0.0.0.0

pip install vllm

python -m vllm.entrypoints.openai.api_server \
  --model ./my-trained-model \
  --port 8000

docker run --gpus all -p 8080:80 \
  -v ./my-model:/model \
  ghcr.io/huggingface/text-generation-inference:latest \
  --model-id /model

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="dummy"  # Not needed for local
)

response = client.chat.completions.create(
    model="my-model",
    messages=[
        {"role": "user", "content": "Hello!"}
    ]
)

FROM python:3.10-slim

WORKDIR /app

# Install dependencies
RUN pip install aitraining torch

# Expose port
EXPOSE 7860

# Run chat server
CMD ["aitraining", "chat", "--host", "0.0.0.0", "--port", "7860"]

docker build -t my-model-server .
docker run -p 7860:7860 my-model-server

docker run --gpus all -p 7860:7860 my-model-server

hey -n 100 -c 10 \
  -m POST \
  -H "Content-Type: application/json" \
  -d '{"prompt": "Hello", "max_tokens": 50}' \
  http://localhost:8000/generate

# locustfile.py
from locust import HttpUser, task

class ModelUser(HttpUser):
    @task
    def generate(self):
        self.client.post("/generate", json={
            "prompt": "Hello, how are you?",
            "max_tokens": 50
        })

locust -f locustfile.py --host http://localhost:8000

aitraining api --port 8000 2>&1 | tee server.log