Add makefile and cloudbuild file to build and push lora-syncer

Signed-off-by: Kunjan <[email protected]>

Author: coolkp

pkg/manifests/vllm/deployment-with-syncer.yaml

@@ -43,18 +43,8 @@ spec:
           - "--max-cpu-loras"
           - "12"
           - "--lora-modules"
-          - '{"name": "sql-lora-0", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-1", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-2", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-3", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-4", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
           - '{"name": "tweet-summary-0", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
           - '{"name": "tweet-summary-1", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-2", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-3", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-4", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
           env:
             - name: PORT
               value: "8000"
@@ -143,16 +133,13 @@ data:
         ensureExist:
           models:
           - base-model: meta-llama/Llama-2-7b-hf
-            id: sql-lora-v1
-            source: yard1/llama-2-7b-sql-lora-test
+            id: tweet-summary-0
+            source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm
           - base-model: meta-llama/Llama-2-7b-hf
-            id: sql-lora-v3
-            source: yard1/llama-2-7b-sql-lora-test
-          - base-model: meta-llama/Llama-2-7b-hf
-            id: sql-lora-v4
-            source: yard1/llama-2-7b-sql-lora-test
+            id: tweet-summary-1
+            source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm
         ensureNotExist:
           models:
           - base-model: meta-llama/Llama-2-7b-hf
-            id: sql-lora-v2
-            source: yard1/llama-2-7b-sql-lora-test
+            id: tweet-summary-2
+            source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm

pkg/manifests/vllm/deployment.yaml

@@ -43,18 +43,8 @@ spec:
           - "--max-cpu-loras"
           - "12"
           - "--lora-modules"
-          - '{"name": "sql-lora-0", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-1", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-2", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-3", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora-4", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
           - '{"name": "tweet-summary-0", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
           - '{"name": "tweet-summary-1", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-2", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-3", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary-4", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
-          - '{"name": "sql-lora", "path": "yard1/llama-2-7b-sql-lora-test", "base_model_name": "llama-2"}'
-          - '{"name": "tweet-summary", "path": "vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm", "base_model_name": "llama-2"}'
           env:
             - name: PORT
               value: "8000"

site-src/guides/dynamic-lora.md

@@ -29,33 +29,40 @@ Rest of the steps are same as [general setup](https://github.com/kubernetes-sigs
         name: dynamic-lora-config
         data:
         configmap.yaml: |
-            vLLMLoRAConfig:
-            ensureExist:   
-                models:
-                - id: tweet-summary-v1
-                    source: tweet-summary-1=/adapters/vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm_1
-                - id: tweet-summary-v2
-                    source: tweet-summary-2=/adapters/vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm_2
+             vLLMLoRAConfig:
+                name: sql-loras-llama
+                port: 8000
+                ensureExist:
+                    models:
+                    - base-model: meta-llama/Llama-2-7b-hf
+                      id: tweet-summary-0
+                      source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm
+                    - base-model: meta-llama/Llama-2-7b-hf
+                      id: tweet-summary-1
+                      source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm
     ```
 
-2. Configure a canary rollout with traffic split using InferenceModel. In this example, 10% of traffic to the chatbot model will be sent to `tweet-summary-3`.
+2. Configure a canary rollout with traffic split using LLMService. In this example, 40% of traffic for tweet-summary model will be sent to the ***tweet-summary-2*** adapter .
 
 ``` yaml
 model:
-    name: chatbot
+    name: tweet-summary
     targetModels:
-    targetModelName: chatbot-v1
-            weight: 90
-    targetModelName: chatbot-v2
+    targetModelName: tweet-summary-0
             weight: 10
+    targetModelName: tweet-summary-1
+            weight: 40
+    targetModelName: tweet-summary-2
+            weight: 40
+    
 ```
             
 3. Finish rollout by setting the traffic to the new version 100%.
 ```yaml
 model:
-    name: chatbot
+    name: tweet-summary
     targetModels:
-    targetModelName: chatbot-v2
+    targetModelName: tweet-summary-2
             weight: 100
 ```
     
@@ -68,12 +75,19 @@ model:
     data:
     configmap.yaml: |
             vLLMLoRAConfig:
-            ensureExist:
-            models:
-            - id: chatbot-v2
-                source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v2
-            ensureNotExist: # Explicitly unregisters the adapter from  model servers
-            models:
-            - id: chatbot-v1
-                source: gs://[TEAM-A-MODELS-BUCKET]/chatbot-v1
+                name: sql-loras-llama
+                port: 8000
+                ensureExist:
+                    models:
+                    - base-model: meta-llama/Llama-2-7b-hf
+                      id: tweet-summary-2
+                      source: vineetsharma/qlora-adapter-Llama-2-7b-hf-TweetSumm
+                ensureNotExist:
+                    models:
+                    - base-model: meta-llama/Llama-2-7b-hf
+                      id: tweet-summary-1
+                      source: gs://[HUGGING FACE PATH]
+                    - base-model: meta-llama/Llama-2-7b-hf
+                      id: tweet-summary-0
+                      source: gs://[HUGGING FACE PATH]
 ```