class OurLLM(CustomLLM): context_window: int = 3900 num_output: int = 256 model_name: str = "custom" dummy_response: str = "My response" @property def metadata(self) -> LLMMetadata: """Get LLM metadata.""" return LLMMetadata( context_window=self.context_window, num_output=self.num_output, model_name=self.model_name, ) @llm_completion_callback() def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse: response = requests.post(url, payload) # Just add your url and payload and other needed stuff in here return CompletionResponse(text=response.text) @llm_completion_callback() def stream_complete( self, prompt: str, **kwargs: Any ) -> CompletionResponseGen: response = "" for token in self.dummy_response: response += token yield CompletionResponse(text=response, delta=token) # define our LLM Settings.llm = OurLLM()
from llama_index.core import Settings Settings.llm = OpenAI(your details) # This will define your llm globally! # every where your llm is going to be used. response = index.as_query_engine.query("your_query")
pip install llama-index-llms-openai
from llama_index.llms.openai import OpenAI