Sentence splitter

from llama_index.text_splitter import TokenTextSplitter

import tiktoken
from llama_index.text_splitter import TokenTextSplitter

text_splitter = TokenTextSplitter(
  separator=" ",
  chunk_size=1024,
  chunk_overlap=20,
  backup_separators=["\n"],
  tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode
)

node_parser = SimpleNodeParser.from_defaults(text_splitter=text_splitter)

>>> print(text)
Here is my todo list:
1. Clean
2. Shop
3. Rest

>>> splits = SentenceSplitter(chunk_size=10, chunk_overlap=0).split_text(text)
>>> splits
['Here is my todo list:\n1.', 'Clean\n2.Shop\n3.Rest']
>>> 

long_text = “Well, Prince, so Genoa and Lucca are now just family estates of the Buonapartes. But I warn you, if you don’t tell me that this means war, if you still try to defend the infamies and horrors perpetrated by that Antichrist—I really believe he is Antichrist—I will have nothing more to do with you and you are no longer my friend, no longer my ‘faithful slave,’ as you call yourself! But how do you do? I see I have frightened you—sit down and tell me all the news.”

tokenizer = AutoTokenizer.from_pretrained("thenlper/gte-small")
sent_splitter = SentenceSplitter(tokenizer=tokenizer.tokenize, chunk_size=128, chunk_overlap=100)
default_splitter = get_default_text_splitter()


node_parser = SimpleNodeParser.from_defaults(text_splitter=sent_splitter)

nodes = get_nodes_from_document(Document(text=long_text), default_splitter,
                                include_metadata=False,
                                include_prev_next_rel=False
)
texts = [node.text for node in nodes]
print(texts)

>>> from llama_index.text_splitter import SentenceSplitter
>>> text = """Well, Prince, so Genoa and Lucca are now just family estates of the Buonapartes.\n But I warn you, if you don’t tell me that this means war, if you still try to defend the infamies and horrors perpetrated by that Antichrist—I really believe he is Antichrist—I will have nothing more to do with you and you are no longer my friend, no longer my ‘faithful slave,’ as you call yourself! But how do you do? I see I have frightened you—sit down and tell me all the news."""
>>> splits = SentenceSplitter(chunk_size=25, chunk_overlap=0).split_text(text)
>>> splits
['Well, Prince, so Genoa and Lucca are now just family estates of the Buonapartes.', 'But I warn you, if you don’t tell me that this means war, if you still try to defend the', 'infamies and horrors perpetrated by that Antichrist—I really believe he is Antichrist—I will have nothing', 'more to do with you and you are no longer my friend, no longer my ‘faithful slave,', '’ as you call yourself! But how do you do?', 'I see I have frightened you—sit down and tell me all the news.']
>>> splits = SentenceSplitter(chunk_size=50, chunk_overlap=0).split_text(text)
>>> splits
['Well, Prince, so Genoa and Lucca are now just family estates of the Buonapartes.\n But I warn you, if you don’t tell me that this means war,', 'if you still try to defend the infamies and horrors perpetrated by that Antichrist—I really believe he is Antichrist—I will have nothing more to do with you and you are no longer my friend,', 'no longer my ‘faithful slave,’ as you call yourself! But how do you do? I see I have frightened you—sit down and tell me all the news.']
>>> 

sent_splitter = SentenceSplitter(tokenizer=tokenizer.tokenize, chunk_size=60, chunk_overlap=20)