import pandas as pd
import numpy as np
import re
import nltk
import matplotlib.pyplot as plt
For TF-IDF and classification
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, classification_report
For LDA topic modeling
from sklearn.decomposition import LatentDirichletAllocation
For BERT/BioBERT classification
from transformers import pipeline
Download NLTK stopwords if needed
nltk.download('stopwords')
from nltk.corpus import stopwords
=========================
1. Load and Preprocess Data
=========================
Load the CSV file (update filename/path as needed)
data = pd.read_csv("NOTEEVENTS.csv")
print("Dataset Columns:", data.columns.tolist())
We will use the 'text' column for clinical notes and 'category' for classification labels.
If you prefer a different label, adjust accordingly.
if 'text' not in data.columns:
raise KeyError("Column 'text' not found in the dataset.")
if 'category' not in data.columns:
raise KeyError("Column 'category' not found in the dataset. This column will be used as the classification label.")
(Optional) Combine 'chartdate' and 'charttime' into a single datetime column
if 'chartdate' in data.columns and 'charttime' in data.columns:
data['datetime'] = pd.to_datetime(data['chartdate'] + ' ' + data['charttime'], errors='coerce')
else:
data['datetime'] = pd.to_datetime(data['chartdate'], errors='coerce') if 'chartdate' in data.columns else pd.NaT
Basic text preprocessing function
def preprocess_text(text):
text = text.lower() # Lowercase
text = re.sub(r'[^a-z0-9\s]', '', text) # Remove punctuation and non-alphanumeric characters
text = re.sub(r'\s+', ' ', text).strip() # Remove extra whitespace
return text
Apply preprocessing to the clinical notes in the 'text' column
data['processed_text'] = data['text'].astype(str).apply(preprocess_text)
=========================
2. TF-IDF + Logistic Regression / SVM Classification
=========================
We use the 'category' column as the label for supervised classification.
X = data['processed_text']
y = data['category']
Split data into training and testing sets (80/20 split; stratify ensures balanced classes)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42, stratify=y
)
Initialize TF-IDF Vectorizer with NLTK stopwords and limit features for efficiency
tfidf = TfidfVectorizer(stop_words=stopwords.words('english'), max_features=5000)
X_train_tfidf = tfidf.fit_transform(X_train)
X_test_tfidf = tfidf.transform(X_test)
----- Logistic Regression -----
lr_model = LogisticRegression(max_iter=200)
lr_model.fit(X_train_tfidf, y_train)
y_pred_lr = lr_model.predict(X_test_tfidf)
print("\nLogistic Regression Classification Report:")
print(classification_report(y_test, y_pred_lr))
----- SVM Classification -----
svm_model = SVC(kernel='linear', probability=True)
svm_model.fit(X_train_tfidf, y_train)
y_pred_svm = svm_model.predict(X_test_tfidf)
print("\nSVM Classification Report:")
print(classification_report(y_test, y_pred_svm))
=========================
3. BERT / BioBERT Classification
=========================
Using Hugging Face pipeline for text classification.
For demonstration, we are using a generic fine-tuned model.
To use BioBERT, replace with an appropriate model identifier, e.g., "dmis-lab/biobert-base-cased-v1.1"
try:
text_classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
# Sample a few clinical notes for demonstration
sample_texts = data['text'].dropna().sample(5, random_state=42).tolist()
bert_results = text_classifier(sample_texts)
print("\nBERT Classification Results (sample predictions):")
for i, (text, result) in enumerate(zip(sample_texts, bert_results), start=1):
print(f"Sample {i} - Text snippet: {text[:100]}...\nPrediction: {result}\n")
except Exception as e:
print("Error with BERT/BioBERT classification pipeline:", e)
=========================
4. LDA Topic Modeling
=========================
Use CountVectorizer for LDA topic modeling on the processed text
vectorizer = CountVectorizer(stop_words=stopwords.words('english'), max_features=5000)
dtm = vectorizer.fit_transform(data['processed_text'])
Set the number of topics to extract (adjust num_topics as needed)
num_topics = 5
lda_model = LatentDirichletAllocation(n_components=num_topics, random_state=42)
lda_model.fit(dtm)
Function to display topics with their top words
def display_topics(model, feature_names, no_top_words):
topics = {}
for topic_idx, topic in enumerate(model.components_):
topics[topic_idx] = [feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]
return topics
no_top_words = 10
topics = display_topics(lda_model, vectorizer.get_feature_names_out(), no_top_words)
print("\nLDA Topics:")
for topic_idx, words in topics.items():
print(f"Topic {topic_idx}: {', '.join(words)}")
(Optional) Visualize topic distribution for a sample document
doc_topic_distribution = lda_model.transform(dtm)
sample_idx = 0
print("\nTopic distribution for sample document:")
for topic_num, prob in enumerate(doc_topic_distribution[sample_idx]):
print(f"Topic {topic_num}: {prob:.4f}")
