from llama index import

AAmardeep

from llama_index import SimpleDirectoryReader, GPTVectorStoreIndex, LLMPredictor, PromptHelper, ServiceContext, StorageContext, load_index_from_storage
from langchain import OpenAI
import os
import openai
import gradio as gr

os.environ["OPENAI_API_KEY"] = "sjksajdkjadskcj"

openai.api_key = os.environ["OPENAI_API_KEY"]

def create_index(path):
max_input = 4096
tokens = 4096
chunk_size = 600
max_chunk_overlap = 1

promptHelper = PromptHelper(max_input, tokens, max_chunk_overlap, chunk_size_limit=chunk_size)
llmPredictor = LLMPredictor(llm=OpenAI(model_name="text-davinci-002", max_tokens=tokens))

docs = SimpleDirectoryReader(path).load_data()

service_context = ServiceContext.from_defaults(llm_predictor=llmPredictor, prompt_helper=promptHelper)

vectorIndex = GPTVectorStoreIndex.from_documents(documents=docs, service_context=service_context)

vectorIndex.storage_context.persist(persist_dir='Store') # issue in this line it over ride pre existing vector data i dont want to create vector of all document when a single document come in my Data directory.
return vectorIndex

create_index('Data')

def answerMe(question):
question = question
storage_context = StorageContext.from_defaults(persist_dir='Store')
index = load_index_from_storage(storage_context)
query_engine = index.as_query_engine()
response = query_engine.query(question)
return response

answer('my questions?')

19 comments

LLogan M

Whats the issue here?

LLogan M

One thing is that you should add the service_context when loading the index

LLogan M

index = load_index_from_storage(storage_context, service_context=service_context)

AAmardeep

vectorIndex.storage_context.persist(persist_dir='Store') # issue in this line it over ride pre existing vector data i dont want to create vector of all document when a single document come in my Data directory.

AAmardeep

Please let me know how can i deal with this ?

LLogan M

persist will always overwrite

If you want to add more documents, use index.insert(document) for each new document you want to add to the index 👍

AAmardeep

it will also store index of my old document ?

AAmardeep

or over ride the vector of old document

LLogan M

insert will keep your old data if you loaded the index

LLogan M

then when you call persist again, all data will go back to disk, included the new data you inserted

LLogan M

Plain Text

index = load_index_from_storage(...)
index.insert(document)
index.storage_context.persist(persist_dir="./store")

AAmardeep

Please wait, i am checking out this

WWhiteFang_Jr

Just answered the same here also

https://discord.com/channels/1059199217496772688/1128005924238409859/1128030258348703764

AAmardeep

Hi @Logan M I am modify my code like this

def answerMe(question):

storage_context = StorageContext.from_defaults(persist_dir = 'Store')
index = load_index_from_storage(storage_context)
docs2 = SimpleDirectoryReader('Data').load_data()
index.insert(docs2)
storage_context = StorageContext.from_defaults(persist_dir = 'Store')
query_engine = index.as_query_engine()
response = query_engine.query(question)
return response

it will occur error

LLogan M

docs2 is a list, you need to iterate

Plain Text

for doc in docs2:
  index.insert(doc)
index.storage_context.persist(persist_dir="Store")

AAmardeep

ok sir let me try

AAmardeep

Sir it's working I am grateful for your invaluable assistance and support with my coding endeavors. Your guidance and expertise have played a pivotal role in my growth as a programmer, and I am immensely grateful for your help.

AAmardeep

Thank you so much

AAmardeep

could you send me any reference link for llama-index module soi can learn about each function and property of llama-index module 🙏

Add a reply

Find answers from the community

from llama index import