feat(visualqa): updated model paramters and file name in igestion

Author: Dinesh Sajwan

lambda/aws-qa-appsync-opensearch/question_answering/src/qa_agent/chain.py

@@ -101,8 +101,8 @@ def run_qa_agent_rag_no_memory(input_params):
     logger.info("starting qa agent with rag approach without memory :: {input_params}")
 
     base64_bytes = input_params['question'].encode("utf-8")
-    model_id = input_params['embeddings_model']['modelId']
-    print(f'model id :: {model_id}')
+    embedding_model_id = input_params['embeddings_model']['modelId']
+    qa_model_id = input_params['qa_model']['modelId']
     sample_string_bytes = base64.b64decode(base64_bytes)
     decoded_question = sample_string_bytes.decode("utf-8")
 
@@ -129,7 +129,7 @@ def run_qa_agent_rag_no_memory(input_params):
                                               os.environ.get('OPENSEARCH_DOMAIN_ENDPOINT'),
                                               os.environ.get('OPENSEARCH_INDEX'),
                                               os.environ.get('OPENSEARCH_SECRET_ID'),
-                                              model_id)
+                                              embedding_model_id)
 
     else:
         logger.info("_retriever already exists")

lambda/aws-qa-appsync-opensearch/question_answering/src/qa_agent/image_qa.py

@@ -33,11 +33,7 @@ def run_qa_agent_on_image_no_memory(input_params):
     }
     send_job_status(status_variables)
 
-    # 1 : load the document
-    global _file_content
-    global _current_file_name
-
-    bucket_name = os.environ['INPUT_BUCKET']
+    
     filename = input_params['filename']
     image_url = input_params['presignedurl']
     logger.info(f"Generating response to question for file {filename}")
@@ -52,15 +48,8 @@ def run_qa_agent_on_image_no_memory(input_params):
         send_job_status(status_variables)
         return
 
-    # 2 : run the question
     streaming = input_params.get("streaming", False)
-    # TODO use streaming in response
-    callback_manager = [StreamingCallbackHandler(status_variables)] if streaming else None
-   
-    #_qa_llm = get_llm(callback_manager,"HuggingFaceM4/idefics-80b-instruct")
-    #TODO : Update get_llm to support sagemaker as provider,
-    # this needs to be updated with @alain changes
-    print(f' get LLM Ideficsllm')
+    callback_manager = [StreamingCallbackHandler(status_variables)] if streaming else None  
     _qa_llm = Ideficsllm.sagemakerendpoint_llm("idefics")
 
     if (_qa_llm is None):
@@ -71,8 +60,6 @@ def run_qa_agent_on_image_no_memory(input_params):
         send_job_status(status_variables)
         return status_variables
 
-    # 3: run LLM
-    #template="User:{question}![]({image})<end_of_utterance>\nAssistant:"
     template = """\n\nUser: {question}![]({image})<end_of_utterance>
          \n\nAssistant:"""
     prompt = PromptTemplate(template=template, input_variables=["image", "question"])
@@ -82,7 +69,6 @@ def run_qa_agent_on_image_no_memory(input_params):
         logger.info(f'image is: {filename}')
         logger.info(f'decoded_question is: {decoded_question}')
         tmp = chain.predict(image=image_url, question=decoded_question)
-        #answer = tmp.removeprefix(' ')
         answer=tmp.split("Assistant:",1)[1]
 
         logger.info(f'tmp is: {tmp}')

lambda/aws-rag-appsync-stepfn-opensearch/embeddings_job/src/lambda.py

@@ -199,7 +199,7 @@ def handler(event,  context: LambdaContext) -> dict:
             if(extension == '.jpg' or extension == '.jpeg' or extension == '.png' or extension == '.svg'):
                 # Try adding text to document
                 #image_detal_file is created by aws rekognition
-                img_load = image_loader(bucket_name, f"{name}-resized{extension}",f"{name}.txt")
+                img_load = image_loader(bucket_name, filename,f"{name}.txt")
                 sub_docs = img_load.load()
                 for doc in sub_docs:
                     doc.metadata['source'] = filename

lambda/aws-rag-appsync-stepfn-opensearch/input_validation/src/lambda.py

@@ -62,7 +62,7 @@ def append_job_info(response, job_id, ignore_existing,modelid):
 
 @tracer.capture_method
 def isvalid_file_format(file_name: str) -> bool:
-    file_format = ['.pdf','.txt','.jpg','.jpeg','.png','.csv','.docx','.ppt','.html','.jpeg']
+    file_format = ['.pdf','.txt','.jpg','.jpeg','.png','.svg']
     if file_name.endswith(tuple(file_format)):
         print(f'valid file format :: {file_format}')
         return True