deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: llama-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: llama-server
  template:
    metadata:
      labels:
        app: llama-server
        ai.gke.io/model: LLaMA2_7B
        ai.gke.io/inference-server: vllm
        examples.ai.gke.io/source: model-garden
    spec:
      shareProcessNamespace: true
      containers:
      - name: inference-server
        image: vllm/vllm-openai:v0.6.3.post1
        resources:
          requests:
            cpu: 5
            memory: 20Gi
            ephemeral-storage: 40Gi
            nvidia.com/gpu : 1
          limits:
            cpu: 5
            memory: 20Gi
            ephemeral-storage: 40Gi
            nvidia.com/gpu : 1
        command: ["/bin/sh", "-c"]
        args:
        - vllm serve meta-llama/Llama-2-7b-hf
        - --host=0.0.0.0
        - --port=8000
        - --tensor-parallel-size=1
        - --swap-space=16
        - --gpu-memory-utilization=0.95
        - --max-model-len=2048
        - --max-num-batched-tokens=4096
        - --disable-log-stats
        - --enable-loras
        - --max-loras=5
        env:
        - name: DEPLOY_SOURCE
          value: UI_NATIVE_MODEL
        - name: MODEL_ID
          value: "Llama2-7B"
        - name: AIP_STORAGE_URI
          value: "gs://vertex-model-garden-public-us/llama2/llama2-7b-hf"
        - name: VLLM_ALLOW_RUNTIME_LORA_UPDATING
          value: "true"
        volumeMounts:
        - mountPath: /dev/shm
          name: dshm
      initContainers:
        - name: configmap-reader-1
          image: us-docker.pkg.dev/kunjanp-gke-dev-2/lora-sidecar/sidecar:latest
          restartPolicy: Always
          env: 
            DYNAMIC_LORA_ROLLOUT_CONFIG: "/config/configmap.yaml"
          volumeMounts:
          - name: config-volume
            mountPath:  /config/configmap.yaml 
            subPath: configmap.yaml
      volumes:
      - name: dshm
        emptyDir:
          medium: Memory
      - name: config-volume
        configMap:
          name: dynamic-lora-config
      nodeSelector:
        cloud.google.com/gke-accelerator: nvidia-l4
        cloud.google.com/gke-nodepool: dynamic-lora

---
apiVersion: v1
kind: Service
metadata:
  name: llama-service
spec:
  selector:
    app: llama-server
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 8000
    targetPort: 8000