Practice: NLP Text Pipeline

import re

def analyze_text(text):
    """Complete text analysis pipeline."""
    
    # Tokenization
    words = text.lower().split()
    clean_words = [re.sub(r"[^\w]", "", w) for w in words if re.sub(r"[^\w]", "", w)]
    
    # Vocabulary statistics
    unique_words = set(clean_words)
    word_freq = {}
    for word in clean_words:
        word_freq[word] = word_freq.get(word, 0) + 1
    
    # Top words (excluding stopwords)
    stopwords = {"the", "a", "an", "is", "it", "in", "on", "at", "to", "was", "and", "or"}
    content_words = {w: c for w, c in word_freq.items() if w not in stopwords and len(w) > 2}
    top_words = sorted(content_words.items(), key=lambda x: x[1], reverse=True)[:5]
    
    # Sentiment
    pos_words = {"great", "excellent", "amazing", "love", "perfect", "fantastic", "good", "clear"}
    neg_words = {"terrible", "bad", "awful", "hate", "worst", "poor", "confusing", "difficult"}
    pos_count = sum(1 for w in clean_words if w in pos_words)
    neg_count = sum(1 for w in clean_words if w in neg_words)
    sentiment = "Positive" if pos_count > neg_count else ("Negative" if neg_count > pos_count else "Neutral")
    
    # Summary
    return {
        "word_count": len(clean_words),
        "unique_words": len(unique_words),
        "avg_word_length": sum(len(w) for w in clean_words) / len(clean_words),
        "top_words": top_words,
        "sentiment": sentiment,
        "pos_words_found": pos_count,
        "neg_words_found": neg_count,
    }

# Analyze sample texts
texts = [
    "Artificial Intelligence is amazing and I love learning about machine learning and deep learning.",
    "This course has a confusing structure and difficult explanations that are hard to follow.",
]

for text in texts:
    print(f"Text: '{text[:60]}...'")
    results = analyze_text(text)
    print(f"  Words: {results['word_count']} total, {results['unique_words']} unique")
    print(f"  Avg word length: {results['avg_word_length']:.1f} chars")
    print(f"  Top content words: {[w for w, _ in results['top_words']]}")
    print(f"  Sentiment: {results['sentiment']}")
    print()