vLLM, SGLang, Ollama & Kubernetes GPU 설정 컴파일러

vllm serve Qwen/Qwen3-32B \
  --tensor-parallel-size 1 \
  --max-model-len 32768 \
  --max-num-seqs 16 \
  --gpu-memory-utilization 0.95 \
  --quantization fp8 \
  --kv-cache-dtype fp8 \
  --enable-prefix-caching \
  --port 8000

python3 -m sglang.launch_server \
  --model-path Qwen/Qwen3-32B \
  --tp 1 \
  --context-length 32768 \
  --mem-fraction-static 0.95 \
  --kv-cache-dtype fp8_e4m3 \
  --max-running-requests 16 \
  --host 0.0.0.0 \
  --port 30000

# Save as Modelfile, then run: ollama create qwen3-32b-custom -f Modelfile
FROM qwen3:32b
PARAMETER num_ctx 32768
PARAMETER num_batch 512
PARAMETER flash_attn true

# Environment variables for ollama serve:
# OLLAMA_NUM_PARALLEL=16 OLLAMA_FLASH_ATTENTION=1 OLLAMA_KV_CACHE_TYPE=q8_0 ollama serve

llama-server \
  -hf Qwen/Qwen3-32B:FP8_E4M3 \
  --ctx-size 524288 \
  --parallel 16 \
  --n-gpu-layers 999 \
  --flash-attn \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  --host 0.0.0.0 --port 8080

services:
  vllm-qwen3-32b:
    image: vllm/vllm-openai:latest
    ipc: host
    ports:
      - "8000:8000"
    environment:
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    command: >
      --model Qwen/Qwen3-32B --tensor-parallel-size 1 --max-model-len 32768 --max-num-seqs 16 --gpu-memory-utilization 0.95 --quantization fp8 --kv-cache-dtype fp8 --enable-prefix-caching
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: qwen3-32b-vllm
  labels:
    app: qwen3-32b-vllm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: qwen3-32b-vllm
  template:
    metadata:
      labels:
        app: qwen3-32b-vllm
    spec:
      volumes:
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: 16Gi
      containers:
        - name: vllm
          image: vllm/vllm-openai:latest
          args:
            - "--model"
            - "Qwen/Qwen3-32B"
            - "--tensor-parallel-size"
            - "1"
            - "--max-model-len"
            - "32768"
            - "--max-num-seqs"
            - "16"
            - "--gpu-memory-utilization"
            - "0.95"
            - "--kv-cache-dtype"
            - "fp8"
          ports:
            - containerPort: 8000
          volumeMounts:
            - name: dshm
              mountPath: /dev/shm
          resources:
            limits:
              nvidia.com/gpu: "1"
              memory: "48Gi"
            requests:
              nvidia.com/gpu: "1"
              cpu: "8"
              memory: "32Gi"