The Value of Text Analytics

Unstructured text contains valuable insights. Customer reviews, support tickets, social media, and documents hold information that can drive business decisions when properly analyzed.

Sentiment Analysis

Using Transformers

from transformers import pipeline

sentiment_analyzer = pipeline(
    "sentiment-analysis",
    model="distilbert-base-uncased-finetuned-sst-2-english"
)

def analyze_sentiment(texts):
    results = sentiment_analyzer(texts)
    return [
        {
            'text': text,
            'sentiment': r['label'],
            'confidence': r['score']
        }
        for text, r in zip(texts, results)
    ]

# Analyze customer reviews
reviews = [
    "Great product, works perfectly!",
    "Terrible experience, never buying again."
]
sentiments = analyze_sentiment(reviews)

Fine-Tuning for Domain

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=3,
    per_device_train_batch_size=16,
    evaluation_strategy='epoch'
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset
)

trainer.train()

Named Entity Recognition

import spacy

nlp = spacy.load('en_core_web_trf')

def extract_entities(text):
    doc = nlp(text)
    return [
        {
            'text': ent.text,
            'label': ent.label_,
            'start': ent.start_char,
            'end': ent.end_char
        }
        for ent in doc.ents
    ]

# Extract from business document
text = "Apple Inc. reported $394 billion in revenue in 2022. Tim Cook announced new products in California."
entities = extract_entities(text)
# Returns: Apple Inc. (ORG), $394 billion (MONEY), 2022 (DATE), Tim Cook (PERSON), California (GPE)

Topic Modeling

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

# Prepare documents
vectorizer = CountVectorizer(max_df=0.95, min_df=2, stop_words='english')
doc_term_matrix = vectorizer.fit_transform(documents)

# Train LDA model
lda = LatentDirichletAllocation(
    n_components=10,
    random_state=42
)
lda.fit(doc_term_matrix)

# Get topic words
def get_topic_words(model, vectorizer, n_words=10):
    words = vectorizer.get_feature_names_out()
    topics = []
    for topic_idx, topic in enumerate(model.components_):
        top_words = [words[i] for i in topic.argsort()[:-n_words-1:-1]]
        topics.append(top_words)
    return topics

Text Classification

from transformers import AutoTokenizer, AutoModelForSequenceClassification

# Multi-label classification for support tickets
class TicketClassifier:
    def __init__(self):
        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
        self.model = AutoModelForSequenceClassification.from_pretrained(
            'bert-base-uncased',
            num_labels=len(categories)
        )
        
    def classify(self, ticket_text):
        inputs = self.tokenizer(
            ticket_text,
            return_tensors='pt',
            truncation=True
        )
        outputs = self.model(**inputs)
        probs = torch.softmax(outputs.logits, dim=-1)
        return {
            'category': categories[probs.argmax()],
            'confidence': probs.max().item()
        }

Building a Text Analytics Pipeline

class TextAnalyticsPipeline:
    def __init__(self):
        self.sentiment = pipeline('sentiment-analysis')
        self.ner = spacy.load('en_core_web_sm')
        
    def analyze(self, text):
        return {
            'sentiment': self.sentiment(text)[0],
            'entities': self.extract_entities(text),
            'summary': self.summarize(text),
            'keywords': self.extract_keywords(text)
        }

Production Tips

• Pre-process text: lowercase, remove noise
• Handle multiple languages with multilingual models
• Cache model inference for repeated texts
• Monitor for data drift over time
• Use batch processing for high volume

Conclusion

Modern NLP has made text analytics accessible. Start with pre-trained models and fine-tune for your specific domain and use cases.