update constants

Author: kaushikmitr

simulations/llm_ig_simulation/__pycache__/loadbalancer.cpython-311.pyc

@@ -49,7 +49,7 @@ def estimate_avg_latency(self, llmactor, input_size, output_size, include_runnin
         items = llmactor.decode_store.items if include_running_requests else llmactor.decoded_store.items
 
         for item in items:
-            if include_running_requests or self.env.now - item.arrival_time > TTL:
+            if self.env.now - item.arrival_time > TTL:
                 continue  # Skip long-running requests
 
             tokens_in_kv_cache_at_start_of_decode = item.tokens_in_kv_cache_at_start_of_decode or 0
@@ -325,14 +325,14 @@ def find_target_pod(self, routing_type, input_size, output_size,  target_latency
           return target_pod, latency_esimated
 
 
-    def queueing_signal(self, use_overall = True) -> bool:
-      if not use_overall:
+    def queueing_signal(self, routing_type = "smart") -> bool:
+      if routing_type == "smart":
         return self.check_saturations(use_pseudo_kv_cache=True, max_saturation= self.queueing_perc) or self.all_servers_queued()
       else :
         return self.get_overall_pending_tokens_perc() > self.queueing_perc or self.all_servers_queued()
 
-    def dequeueing_signal(self, use_overall = True) -> bool:
-      if not use_overall:
+    def dequeueing_signal(self, routing_type = "smart") -> bool:
+      if routing_type == "smart":
         return self.check_saturations(use_pseudo_kv_cache=True, max_saturation= self.queueing_perc)  == False and  self.all_servers_queued() == False
       else :
         return self.get_overall_pending_tokens_perc() < self.queueing_perc and self.all_servers_queued() == False
@@ -357,11 +357,10 @@ def dequeue(self) -> Optional[Request]:
 
     def dequeue_process(self, routing_type, drop_late_requests = False):
         while True:
-            if not self.check_if_queues_empty() and self.dequeueing_signal():
+            if not self.check_if_queues_empty() and self.dequeueing_signal(routing_type):
                 # Get the request with the highest SLO violation
                 req = self.dequeue()
                 if   req:
-                  #if self.env.now - req.arrival_time > req.target_latency:
                   if (drop_late_requests == False) or (self.env.now - req.arrival_time < 100*req.target_latency): #ad-hoc
                     target_pod, estimated_latency = self.find_target_pod(routing_type, req.input_size, req.output_size, req.target_latency, req.lora)
                     req.target_pod = target_pod.id
@@ -467,7 +466,7 @@ def allPodsRunningCritical(self):
 
     def generate_request_inference_gateway(
         self, rate, lora_requested, target_latency_list, prefix_latency_list, 
-        routing_type="random", prompt_output_tuple=None, mean_request_size=None, 
+        routing_type, prompt_output_tuple=None, mean_request_size=None, 
         std_request_size=None, mean_output_size=None, std_output_size=None, 
         estimated_output_size=None):
       """
@@ -495,7 +494,7 @@ def generate_request_inference_gateway(
         cnt += 1
         self.messages_remaining_cnt -= 1
 
-        if self.should_enqueue_request():
+        if self.should_enqueue_request(routing_type):
             self.enqueue_request(new_req, lora_requested, target_latency)
         else:
             self.route_request(new_req, routing_type, input_size, output_size, target_latency, lora_requested, estimated_output_size)
@@ -517,8 +516,8 @@ def create_request(self, request_id, input_size, output_size, target_latency):
       new_req.target_latency = target_latency
       return new_req
 
-    def should_enqueue_request(self):
-      return self.queueing_signal() or not self.check_if_queues_empty()
+    def should_enqueue_request(self, routing_type):
+      return self.queueing_signal(routing_type) or not self.check_if_queues_empty()
 
     def enqueue_request(self, new_req, lora_requested, target_latency):
       if lora_requested:

simulations/llm_ig_simulation/loadbalancer.py

@@ -7,8 +7,8 @@
 
 def main():
     parser = argparse.ArgumentParser(description="Simulate LLM load balancing with configurable parameters.")
-    parser.add_argument("--rates_lo", nargs='+', type=int, default=[30, ], help="List of low rates.")
-    parser.add_argument("--rates_hi", nargs='+', type=int, default=[30,], help="List of high rates.")
+    parser.add_argument("--rates_lo", nargs='+', type=int, default=[35, 30, 25, 20, 15, 10, 5, 1], help="List of low rates.")
+    parser.add_argument("--rates_hi", nargs='+', type=int, default=[35, 30, 25, 20, 15, 10, 5, 1], help="List of high rates.")
     parser.add_argument("--no_of_messages", type=int, default=2500, help="Number of messages to simulate.")
     parser.add_argument("--mean_request_size_1", type=int, default=202, help="Mean request size for set 1.")
     parser.add_argument("--std_request_size_1", type=int, default=20, help="Standard deviation of request size for set 1.")
@@ -25,6 +25,9 @@ def main():
     parser.add_argument('--prefix-latency-lo', nargs='+', type=float, help='List of prefix of target latencies for low priority requests.')
     parser.add_argument('--prefix-latency-hi', nargs='+', type=float, help='List of prefix of target latencies for high priority requests.')
 
+    
+    parser.add_argument('--number-of-servers',  type=int, default=6, help='List of target latencies for high priority requests.')
+    
     args = parser.parse_args()
 
      # Use provided arguments or defaults
@@ -51,6 +54,8 @@ def main():
 
     prefix_latency_list_lo = args.prefix_latency_lo if args.prefix_latency_lo else ['lo']
     prefix_latency_list_hi = args.prefix_latency_hi if args.prefix_latency_hi else ['hi']
+    
+    number_of_servers = args.number_of_servers
 
     # Define a structure to store results for all routing types
     results = {
@@ -110,7 +115,7 @@ def main():
               'tol_lat_time_lo': [], 'tol_lat_time_hi': []},
 }
 
-    all_routing_types = ["least", ]
+    all_routing_types = ["least", "smart", "random" ]
     prompt_output_tuple = None
 
 # Iterate over routing types
@@ -125,7 +130,6 @@ def main():
 
             # Simpy environment and LLM actors setup
             env = simpy.Environment()
-            number_of_servers =6
             list_of_llmactors = [LLMActor(env, 1, id) for id in range(number_of_servers)]
             lb = LoadBalancer(env, number_of_servers=number_of_servers, list_of_llmactors=list_of_llmactors, req_dict_prefill=req_dict_prefill, req_dict=req_dict, messages_remaining_cnt=no_of_messages*2)
             lb.queueing_perc = queueing_perc