With my streaming response it seems to be missing the first word from the response:
@app.post("/ask")
async def ask(data: AskModel):
s3 = get_s3()
chat_info = get_convo(data.chat_uuid, s3)
access = chat_info.get('meta', {}).get('access', [])
index_set = get_index(access, fs=s3)
chat_history = chat_info["history"]
chat_engine = index_set.as_chat_engine(
similarity_top_k=3,
streaming=True,
chat_history=chat_history,
)
def stream_response():
streaming_response = chat_engine.stream_chat(data.question)
response_gen = streaming_response.response_gen
first_token = next(response_gen) # Get the first token of the response
response = first_token
yield f"data: {response}\n\n" # Yield the first token separately
for token in response_gen:
yield f"data: {response}\n\n"
response+=token
chat_history.extend([
{"role": "user", "content": data.question},
{"role": "assistant", "content": ''.join(response)}
])
chat_info['history'] = chat_history
save_chat(data.chat_uuid, chat_info, s3)
return StreamingResponse(stream_response(), media_type="text/event-stream")
How can i get the make sure the first token is included of the response? :)
the
e.g:
call between the person and the company resulted in the person...
it should clearly have something on front of the
' call' text
with a non stream version, the response looks like:
"response": " The call resulted in the person being informed..."