@Logan M @WhiteFang_Jr Hi, I would like

Hi, I would like to wrap all this as a module in a query pipeline, how can i do it?

from llama_index.core.agent.react_multimodal.step import MultimodalReActAgentWorker
from llama_index.core.agent import AgentRunner
from llama_index.core.multi_modal_llms import MultiModalLLM
from llama_index.multi_modal_llms.openai import OpenAIMultiModal
from llama_index.core.agent import Task
from llama_index.core.schema import ImageDocument

mm_llm = OpenAIMultiModal(model="gpt-4-vision-preview", max_new_tokens=1000)

react_step_engine = MultimodalReActAgentWorker.from_tools(
[], # you can put some tools here
multi_modal_llm=mm_llm,
verbose=True,
)
agent = AgentRunner(react_step_engine)

query_str = "Your query string here"

image_document = ImageDocument(image_path="path_to_your_image.png")

task = agent.create_task(
query_str,
extra_state={"image_docs": [image_document]},
)

def execute_step(agent: AgentRunner, task: Task):
step_output = agent.run_step(task.task_id)
if step_output.is_last:
response = agent.finalize_response(task.task_id)
print(f"> Agent finished: {str(response)}")
return response
else:
return None

def execute_steps(agent: AgentRunner, task: Task):
response = execute_step(agent, task)
while response is None:
response = execute_step(agent, task)
return response

response = execute_steps(agent, task)