Text Processing

# Common text preprocessing steps

import re

def preprocess_text(text, lowercase=True, remove_punctuation=True,
                   remove_stopwords=True, stopwords=None):
    """Apply common text preprocessing steps."""
    
    if stopwords is None:
        stopwords = {"the", "a", "an", "is", "it", "in", "on", "at", "to", "was", "for"}
    
    # Step 1: lowercase
    if lowercase:
        text = text.lower()
    
    # Step 2: remove punctuation
    if remove_punctuation:
        text = re.sub(r"[^\w\s]", "", text)
    
    # Step 3: tokenize (split into words)
    words = text.split()
    
    # Step 4: remove stopwords
    if remove_stopwords:
        words = [w for w in words if w not in stopwords]
    
    return words

# Test preprocessing
raw_texts = [
    "The movie was absolutely fantastic! I loved every second.",
    "I hated it, what a waste of time!!!",
    "It's an OK film, nothing special.",
]

print("Text Preprocessing Results:")
print()
for text in raw_texts:
    tokens = preprocess_text(text)
    print(f"  Original: '{text}'")
    print(f"  Processed: {tokens}")
    print()