Tokenization

# Demonstrating different tokenization strategies

def word_tokenize(text):
    """Simple word tokenizer."""
    return text.split()

def char_tokenize(text):
    """Character-level tokenizer."""
    return list(text)

def simple_subword_tokenize(text, vocab_pieces=None):
    """
    Simplified subword tokenizer illustration.
    Real BPE tokenizers learn piece boundaries from data.
    """
    if vocab_pieces is None:
        # Common subword pieces (learned from large text corpus in real models)
        vocab_pieces = ["un", "re", "ing", "tion", "ed", "pre", "ful", "ly", "ness"]
    
    tokens = []
    for word in text.split():
        tokenized_word = []
        remaining = word.lower()
        for piece in sorted(vocab_pieces, key=len, reverse=True):
            if piece in remaining:
                idx = remaining.find(piece)
                if idx > 0:
                    tokenized_word.append(remaining[:idx])
                tokenized_word.append(piece)
                remaining = remaining[idx + len(piece):]
        if remaining:
            tokenized_word.append(remaining)
        tokens.extend(tokenized_word if tokenized_word else [word])
    
    return tokens

# Compare tokenization approaches
sentences = [
    "running quickly",
    "unbelievable prediction",
    "fantastic",
]

print("Tokenization Comparison:")
print()
for sentence in sentences:
    print(f"Text: '{sentence}'")
    print(f"  Word tokens:    {word_tokenize(sentence)}")
    print(f"  Char tokens:    {char_tokenize(sentence)[:8]}...")
    print(f"  Subword tokens: {simple_subword_tokenize(sentence)}")
    print()

print("Modern LLMs use subword tokenization (BPE) to handle any word.")
print("GPT-4 uses a vocabulary of about 100,000 subword tokens.")