Documents and Data

# Document chunking: splitting text into pieces for RAG
# Install: pip install langchain

def simple_document_chunker(text, chunk_size=500, overlap=50):
    """
    Split a long document into smaller overlapping chunks.

    Why overlap? The answer to a question might span two adjacent chunks.
    Overlapping chunks ensure no important information is cut off at the boundary.

    Parameters:
    - text: the full document text
    - chunk_size: maximum characters per chunk
    - overlap: how many characters each chunk shares with the next
    """
    chunks = []
    start = 0

    while start < len(text):
        # Find the end of this chunk
        end = start + chunk_size

        # If not at the end of the document, try to end at a sentence boundary
        if end < len(text):
            # Look for the last period within the chunk
            last_period = text.rfind('.', start, end)
            if last_period != -1 and last_period > start + chunk_size // 2:
                end = last_period + 1  # Include the period

        chunk = text[start:end].strip()
        if chunk:  # Only add non-empty chunks
            chunks.append(chunk)

        # Move start forward, accounting for overlap
        start = end - overlap

    return chunks

# Example usage
sample_document = """
Generative AI is transforming how businesses operate.
Companies use large language models for customer support, content creation, and data analysis.
These models learn from massive datasets and can generate human-like text.
One key challenge is ensuring AI outputs are accurate and up to date.
Retrieval-Augmented Generation solves this by connecting AI to current knowledge sources.
Businesses that adopt RAG systems see significant improvements in AI accuracy.
The technology works by finding relevant documents and using them to generate responses.
This approach reduces hallucinations and keeps AI grounded in real data.
"""

chunks = simple_document_chunker(sample_document, chunk_size=200, overlap=30)
print(f"Document split into {len(chunks)} chunks:")
print()
for i, chunk in enumerate(chunks, 1):
    print(f"Chunk {i} ({len(chunk)} chars): {chunk[:80]}...")
    print()