Move under tools

Signed-off-by: Kunjan <[email protected]>

Author: coolkp

examples/dynamic-lora-sidecar/.gitignore renamed to tools/dynamic-lora-sidecar/.gitignore

@@ -0,0 +1,18 @@
+vLLMLoRAConfig:
+  name: sql-loras-llama
+  ensureExist:
+    models:
+    - base-model: meta-llama/Llama-2-7b-hf
+      id: sql-lora-v8
+      source: yard1/llama-2-7b-sql-lora-test
+    - base-model: meta-llama/Llama-2-7b-hf
+      id: sql-lora-v12
+      source: yard1/llama-2-7b-sql-lora-test
+  ensureNotExist:
+    models:
+    - base-model: meta-llama/Llama-2-7b-hf
+      id: sql-lora-v2
+      source: yard1/llama-2-7b-sql-lora-test
+    - base-model: meta-llama/Llama-2-7b-hf
+      id: sql-lora-v7
+      source: yard1/llama-2-7b-sql-lora-test

examples/dynamic-lora-sidecar/Dockerfile renamed to tools/dynamic-lora-sidecar/Dockerfile

@@ -0,0 +1,125 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: llama-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: llama-server
+  template:
+    metadata:
+      labels:
+        app: llama-server
+        ai.gke.io/model: LLaMA2_7B
+        ai.gke.io/inference-server: vllm
+        examples.ai.gke.io/source: model-garden
+    spec:
+      shareProcessNamespace: true
+      containers:
+      - name: inference-server
+        image: vllm/vllm-openai:v0.6.3.post1
+        resources:
+          requests:
+            cpu: 5
+            memory: 20Gi
+            ephemeral-storage: 40Gi
+            nvidia.com/gpu : 1
+          limits:
+            cpu: 5
+            memory: 20Gi
+            ephemeral-storage: 40Gi
+            nvidia.com/gpu : 1
+        command: ["/bin/sh", "-c"]
+        args:
+        - vllm serve meta-llama/Llama-2-7b-hf
+        - --host=0.0.0.0
+        - --port=8000
+        - --tensor-parallel-size=1
+        - --swap-space=16
+        - --gpu-memory-utilization=0.95
+        - --max-model-len=2048
+        - --max-num-batched-tokens=4096
+        - --disable-log-stats
+        - --enable-loras
+        - --max-loras=5
+        env:
+        - name: DEPLOY_SOURCE
+          value: UI_NATIVE_MODEL
+        - name: MODEL_ID
+          value: "Llama2-7B"
+        - name: AIP_STORAGE_URI
+          value: "gs://vertex-model-garden-public-us/llama2/llama2-7b-hf"
+        - name: VLLM_ALLOW_RUNTIME_LORA_UPDATING
+          value: "true"
+        - name: HF_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: hf-token  # The name of your Kubernetes Secret
+              key: token   # The specific key within the Secret
+        - name: DYNAMIC_LORA_ROLLOUT_CONFIG
+          value: "/config/configmap.yaml"
+        volumeMounts:
+        - mountPath: /dev/shm
+          name: dshm
+      initContainers:
+        - name: lora-adapter-syncer
+          tty: true
+          stdin: true 
+          image: us-docker.pkg.dev/kunjanp-gke-dev-2/lora-sidecar/sidecar:latest
+          restartPolicy: Always
+          imagePullPolicy: Always
+          env: 
+            - name: DYNAMIC_LORA_ROLLOUT_CONFIG
+              value: "/config/configmap.yaml"
+          volumeMounts: # DO NOT USE subPath
+          - name: config-volume
+            mountPath:  /config
+      volumes:
+      - name: dshm
+        emptyDir:
+          medium: Memory
+      - name: config-volume
+        configMap:
+          name: dynamic-lora-config
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: llama-service
+spec:
+  selector:
+    app: llama-server
+  type: ClusterIP
+  ports:
+  - protocol: TCP
+    port: 8000
+    targetPort: 8000
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dynamic-lora-config
+data:
+  configmap.yaml: |
+      vLLMLoRAConfig:
+        name: sql-loras-llama
+        ensureExist:
+          models:
+          - base-model: meta-llama/Llama-2-7b-hf
+            id: sql-lora-v1
+            source: yard1/llama-2-7b-sql-lora-test
+          - base-model: meta-llama/Llama-2-7b-hf
+            id: sql-lora-v3
+            source: yard1/llama-2-7b-sql-lora-test
+          - base-model: meta-llama/Llama-2-7b-hf
+            id: sql-lora-v4
+            source: yard1/llama-2-7b-sql-lora-test
+        ensureNotExist:
+          models:
+          - base-model: meta-llama/Llama-2-7b-hf
+            id: sql-lora-v2
+            source: yard1/llama-2-7b-sql-lora-test