hey all - having an issue with pickling objects in what seems to be a relatively simple scenario. here's the code:
from llama_index import VectorStoreIndex
from llama_index.schema import Document
import os
from llama_index.node_parser import SimpleNodeParser
from llama_index.text_splitter import TokenTextSplitter
class NewlineTextSplitter(TokenTextSplitter):
def split_text(self, text):
# Split the text into chunks based on newlines
chunks = text.split('\n\n')
return chunks
class CharacterSheetIndexer:
def __init__(self, character_sheets_dir):
self.character_sheets_dir = character_sheets_dir
self.indexes = {}
def create_indexes(self):
# Create a NodeParser that uses NewlineTextSplitter
node_parser = SimpleNodeParser(text_splitter=NewlineTextSplitter())
# Read all character sheets
for filename in os.listdir(self.character_sheets_dir):
with open(os.path.join(self.character_sheets_dir, filename), 'r') as f:
character_sheet = f.read()
# Create a document from the character sheet
document = Document(text=character_sheet)
# Chunk the document into nodes using the NodeParser
nodes = node_parser.parse(document)
# Create an index for the character sheet
self.indexes[filename] = VectorStoreIndex.from_documents(nodes)
def main():
# Create an instance of CharacterSheetIndexer
indexer = CharacterSheetIndexer(character_sheets_dir='resources/characters')
# Index all character sheets
indexer.create_indexes()
if __name__ == "__main__":
main()