llama_index/llama-index-integrations/rea...

   Attributes:
        prefix (str): URL prefix to focus the scraping.
        max_depth (int): Maximum depth for BFS algorithm.

def __init__(self, prefix: str, max_depth: int = 10, excluded_uris: List[str] = None) -> None:

self.excluded_uris = set(excluded_uris) if excluded_uris is not None else set()

reader = WholeSiteReader(prefix="https://comma.ai", max_depth=10, excluded_uris=["/shop"])

def uri_not_excluded(self, uri: str) -> bool:
        for excluded_uri in self.excluded_uris:
            if excluded_uri in uri:
                return False
        return True

if depth > self.max_depth or not self.uri_not_excluded(current_url):
                continue