@Logan M got it to work lol
for idx, node in enumerate(nodes):
train_dataset_generator = DatasetGenerator(
[node],
llm=llm2,
question_gen_query=question_gen_query,
show_progress=True,
)
node_questions_0 = train_dataset_generator.generate_questions_from_nodes(num=2)
for question in tqdm(node_questions_0):
prompt = PromptTemplate(question)
print(f"[Node {idx}] Generated questions:\n {node_questions_0}")
output = llm2.structured_predict(output_cls=Document, prompt=prompt)
print(question)
o a JSON string
json_output = json.dumps(jsonable_encoder(output), ensure_ascii=False)
# Open the file in append mode to add the new entry
with open('output.json', 'a', encoding='utf-8') as f:
if not first_entry:
f.write(',') # Add a comma before the next entry if it's not the first one
else:
first_entry = False # Update the flag after the first entry
f.write(json_output) # Write the JSON string
# After all entries are written, close the list with a ']'
with open('output.json', 'a', encoding='utf-8') as f:
f.write(']')
Unless there's a more efficent way