93 lines
2.6 KiB
YAML
93 lines
2.6 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
component: inference
|
|
template:
|
|
metadata:
|
|
labels:
|
|
component: inference
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
enableServiceLinks: false
|
|
securityContext:
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
nodeSelector:
|
|
feature.node.kubernetes.io/pci-0300_10de.present: "true"
|
|
containers:
|
|
- name: vllm
|
|
image: vllm/vllm-openai:v0.9.2
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
args:
|
|
- --model={{ .model }}
|
|
- --max-model-len={{ .maxModelLen }}
|
|
- --tensor-parallel-size=1
|
|
- --gpu-memory-utilization={{ .gpuMemoryUtilization }}
|
|
- --enforce-eager
|
|
- --reasoning-parser=deepseek_r1
|
|
- --api-key=$(VLLM_API_KEY)
|
|
env:
|
|
- name: VLLM_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vllm-secrets
|
|
key: apiKey
|
|
- name: VLLM_TORCH_DTYPE
|
|
value: "auto"
|
|
- name: VLLM_WORKER_CONCURRENCY
|
|
value: "1"
|
|
- name: HF_HOME
|
|
value: "/root/.cache/huggingface"
|
|
- name: NUMBA_CACHE_DIR
|
|
value: "/tmp"
|
|
volumeMounts:
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
ports:
|
|
- name: http
|
|
containerPort: 8000
|
|
resources:
|
|
requests:
|
|
cpu: "{{ .cpuRequest }}"
|
|
memory: "{{ .memoryRequest }}"
|
|
nvidia.com/gpu: {{ .gpuCount }}
|
|
limits:
|
|
cpu: "{{ .cpuLimit }}"
|
|
memory: "{{ .memoryLimit }}"
|
|
nvidia.com/gpu: {{ .gpuCount }}
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: http
|
|
initialDelaySeconds: 300
|
|
periodSeconds: 30
|
|
timeoutSeconds: 10
|
|
failureThreshold: 5
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: http
|
|
initialDelaySeconds: 1800
|
|
periodSeconds: 60
|
|
timeoutSeconds: 10
|
|
failureThreshold: 3
|
|
volumes:
|
|
- name: hf-cache
|
|
emptyDir: {}
|
|
tolerations:
|
|
- key: "nvidia.com/gpu"
|
|
operator: "Exists"
|
|
effect: "NoSchedule"
|