apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      component: inference
  template:
    metadata:
      labels:
        component: inference
    spec:
      runtimeClassName: nvidia
      enableServiceLinks: false
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      nodeSelector:
        feature.node.kubernetes.io/pci-0300_10de.present: "true"
      containers:
        - name: vllm
          image: vllm/vllm-openai:v0.5.4
          imagePullPolicy: IfNotPresent
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
          args:
            - --model={{ .model }}
            - --max-model-len={{ .maxModelLen }}
            - --tensor-parallel-size=1
            - --gpu-memory-utilization={{ .gpuMemoryUtilization }}
            - --enforce-eager
            - --api-key=$(VLLM_API_KEY)
          env:
            - name: VLLM_API_KEY
              valueFrom:
                secretKeyRef:
                  name: vllm-secrets
                  key: apiKey
            - name: VLLM_TORCH_DTYPE
              value: "auto"
            - name: VLLM_WORKER_CONCURRENCY
              value: "1"
            - name: HF_HOME
              value: "/root/.cache/huggingface"
            - name: NUMBA_CACHE_DIR
              value: "/tmp"
          ports:
            - name: http
              containerPort: 8000
          resources:
            requests:
              cpu: "{{ .cpuRequest }}"
              memory: "{{ .memoryRequest }}"
              nvidia.com/gpu: {{ .gpuCount }}
            limits:
              cpu: "{{ .cpuLimit }}"
              memory: "{{ .memoryLimit }}"
              nvidia.com/gpu: {{ .gpuCount }}
          readinessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 300
            periodSeconds: 30
            timeoutSeconds: 10
            failureThreshold: 5
          livenessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 600
            periodSeconds: 60
            timeoutSeconds: 10
            failureThreshold: 3
      tolerations:
        - key: "nvidia.com/gpu"
          operator: "Exists"
          effect: "NoSchedule"