I'm assuming it's api instability but

openai.log = "debug"

llm = ChatOpenAI(model=model, temperature=0)
memory = ConversationSummaryBufferMemory(
    memory_key="memory",
    return_messages=True,
    llm=llm,
    max_token_limit=100000 if "preview" in model else max_token_limit,
)
agent_kwargs = {
    "extra_prompt_messages": [MessagesPlaceholder(variable_name="memory")],
    "system_message": SystemMessage(
        content="You are a superpowered version of GPT that is able to answer questions about the data you're "
        "connected to. Each different tool you have represents a different dataset to interact with. "
        "If you are asked to perform a task that spreads across multiple datasets, use multiple tools "
        "for the same prompt. When the user types links in chat, you will have already been connected "
        "to the data at the link by the time you respond. When using tools, the input should be "
        "clearly created based on the request of the user. For example, if a user uploads an invoice "
        "and asks how many usage hours of X was present in the invoice, a good query is 'X hours'. "
        "Avoid using single word queries unless the request is very simple. You can query multiple times to break down complex requests and retrieve more information."
    ),
}
agent_chain = initialize_agent(
    tools=tools,
    llm=llm,
    agent=AgentType.OPENAI_FUNCTIONS,
    verbose=True,
    agent_kwargs=agent_kwargs,
    memory=memory,
    handle_parsing_errors="Check your output and make sure it conforms!",
)


My index tools are defined like
tool_config = IndexToolConfig(
    query_engine=engine,
    name=f"{filename}-index",
    description=f"Use this tool if the query seems related to this summary: {summary}",
    tool_kwargs={
        "return_direct": False,
    },
    max_iterations=5,
)
tool = LlamaIndexTool.from_tool_config(tool_config)

Where engine is defined as
retriever = VectorIndexRetriever(
    index=index, similarity_top_k=2, service_context=service_context
)

response_synthesizer = get_response_synthesizer(
    response_mode=ResponseMode.COMPACT_ACCUMULATE,
    use_async=True,
    refine_template=TEXT_QA_SYSTEM_PROMPT,
    service_context=service_context,
    verbose=True,
)

engine = RetrieverQueryEngine(
    retriever=retriever, response_synthesizer=response_synthesizer
)

from llama_index.llms import OpenAI

llm = OpenAI(temeprature=0.1, model="gpt-3.5-turbo")

timeout = httpx.Timeout(10.0, read=5.0, write=10.0, connect=2.0)
service_context = ServiceContext.from_defaults(
    llm=OpenAI(max_retries=2, additional_kwargs={"timeout": timeout}),
    embed_model=embedding_model,
    callback_manager=callback_manager,
    node_parser=node_parser,
)

    def index_file(
        self, file_path, service_context, suffix=None
    ) -> GPTVectorStoreIndex:
        if suffix and suffix == ".md":
            loader = MarkdownReader()
            document = loader.load_data(file_path)
        elif suffix and suffix == ".epub":
            epub_loader = EpubReader()
            document = epub_loader.load_data(file_path)
        else:
            document = SimpleDirectoryReader(input_files=[file_path]).load_data()
        index = GPTVectorStoreIndex.from_documents(
            document, service_context=service_context, use_async=True
        )
        return index

document = SimpleDirectoryReader(input_files=[file_path]).load_data()
print(len(document))

service_context = ServiceContext.from_defaults(...., embed_model="local:BAAI/bge-small-en-v1.5")