Self-Hosted LLM Setup

# macOS / Linux
curl -fsSL https://ollama.com/install.sh | sh

# macOS (Homebrew)
brew install ollama

# Recommended for tool calling (AI assistant features)
ollama pull qwen2.5:7b       # 4.7GB, fast, good at tool calling
ollama pull llama3.1          # 4.7GB, general purpose

# For better quality (needs 16GB+ RAM)
ollama pull qwen2.5:14b      # 9GB, excellent quality
ollama pull llama3.1:70b      # 40GB, near-GPT-4 quality (needs 64GB RAM)

ollama serve

# Set AI provider to Ollama. The PUT body takes `settings` as an ARRAY of
# { key, value } pairs (an object map is rejected with 400). Requires an
# admin session/API key; in multi-tenant mode only the platform-owner tenant
# may write platform settings.
curl -X PUT http://localhost:3000/api/v1/platform-settings \
  -H "Content-Type: application/json" \
  -H "X-Requested-With: XMLHttpRequest" \
  -H "X-API-Key: <admin-api-key>" \
  -d '{
    "category": "ai",
    "settings": [
      { "key": "ai_provider", "value": "ollama" },
      { "key": "ai_model", "value": "qwen2.5:7b" },
      { "key": "ai_base_url", "value": "http://localhost:11434" },
      { "key": "ai_api_key", "value": "", "encrypted": true }
    ]
  }'

# On a GPU instance (e.g., AWS g5.xlarge)
pip install vllm
vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000

docker run --gpus all -p 8080:80 \
  ghcr.io/huggingface/text-generation-inference:latest \
  --model-id meta-llama/Llama-3.1-8B-Instruct

# docker-compose.yml
services:
  kaireon-api:
    image: 422500312304.dkr.ecr.us-east-1.amazonaws.com/kaireon-api:latest
    environment:
      - AI_PROVIDER=ollama
      - AI_MODEL=qwen2.5:7b
      - AI_BASE_URL=http://ollama:11434
    depends_on:
      - ollama

  ollama:
    image: ollama/ollama
    volumes:
      - ollama_data:/root/.ollama
    # Pull model on first start:
    # docker exec ollama ollama pull qwen2.5:7b

volumes:
  ollama_data: