from llama_index.finetuning import ( generate_qa_embedding_pairs, EmbeddingQAFinetuneDataset, ) from llama_index.finetuning import SentenceTransformersFinetuneEngine import os os.environ['HF_TOKEN'] = "***" train_dataset = EmbeddingQAFinetuneDataset.from_json("data/li_full.json") val_dataset = EmbeddingQAFinetuneDataset.from_json("data/li_test.json") #Run Embedding Finetuning from llama_index.finetuning import SentenceTransformersFinetuneEngine finetune_engine = SentenceTransformersFinetuneEngine( train_dataset, model_id="jinaai/jina-embeddings-v2-base-en", model_output_path="test_llama_index_finetune", val_dataset=val_dataset, ) finetune_engine.finetune() embed_model = finetune_engine.get_finetuned_model() embed_model