GGML model

#Debug logging
import logging
import sys

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext
from llama_index.node_parser import SimpleNodeParser
from llama_index.llms import LangChainLLM
from langchain.llms import LlamaCpp
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler

required_exts = ['.txt']
documents = SimpleDirectoryReader(
    './docs',
    recursive=True,
    required_exts=required_exts,
).load_data()

parser = SimpleNodeParser()

nodes = parser.get_nodes_from_documents(documents)

model_path = "./models/llama-2-7b-chat.ggmlv3.q4_0.bin"

# Callbacks support token-wise streaming
callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
# Verbose is required to pass to the callback manager

logging.debug(f"Loading model from {model_path}")

llm = LlamaCpp(
    model_path=model_path,
    #input={"temperature": 0.75, "max_length": 2000, "top_p": 1},
    callback_manager=callback_manager,
    verbose=True,
)

logging.debug(f"Wrapping LLM with LangChainLLM")

wrapped_llm = LangChainLLM(
    llm=llm,
    callback_manager=callback_manager
)

logging.debug(f"Creating service context")

service_context = ServiceContext.from_defaults(
    #chunk_size=1024, 
    llm=wrapped_llm,
)

logging.debug(f"Creating index")

index = VectorStoreIndex(
    service_context=service_context,
    nodes=nodes,
    show_progress=True,
)

#index.storage_context.persist(persist_dir="./models/conectar/index")

query_engine = index.as_query_engine()
response = query_engine.query("Terciario es un")
print(response)