Views
No views yet
som_model.pkl: Trained SOM model weights and parameterscluster_assignments.json: Document-to-cluster assignments for all 11,412 recordscluster_analysis.json: Detailed analysis of each cluster including keywords and topicsinteractive_som_map.html: Interactive visualization of the SOM grid with cluster informationpip install numpy scikit-learn matplotlib plotly1import pickle
2import json
3import numpy as np
4from sklearn.metrics.pairwise import cosine_similarity
5
6# Load the trained SOM model
7with open('som_model.pkl', 'rb') as f:
8 som_model = pickle.load(f)
9
10# Load cluster assignments
11with open('cluster_assignments.json', 'r') as f:
12 cluster_assignments = json.load(f)
13
14# Load cluster analysis
15with open('cluster_analysis.json', 'r') as f:
16 cluster_analysis = json.load(f)
17
18# Example: Get cluster for a new document embedding
19def get_cluster_for_embedding(embedding, som_model):
20 """Get the cluster assignment for a new document embedding"""
21 # Find the best matching unit (BMU)
22 bmu = som_model.winner(embedding)
23 return f"{bmu[0]},{bmu[1]}"
24
25# Example: Find similar documents
26def find_similar_documents(embedding, cluster_assignments, top_k=5):
27 """Find similar documents based on cluster membership"""
28 cluster = get_cluster_for_embedding(embedding, som_model)
29
30 # Get all documents in the same cluster
31 cluster_docs = [doc for doc, doc_cluster in cluster_assignments.items()
32 if doc_cluster == cluster]
33
34 return cluster_docs[:top_k]interactive_som_map.html in a web browser to explore the SOM grid interactively. The visualization shows:1# Train a new SOM with different parameters
2from minisom import MiniSom
3
4def train_custom_som(embeddings, grid_size=(20, 20), sigma=1.0, learning_rate=0.1):
5 som = MiniSom(grid_size[0], grid_size[1], embeddings.shape[1],
6 sigma=sigma, learning_rate=learning_rate, random_seed=42)
7 som.train_random(embeddings, 100)
8 return som1def analyze_cluster(cluster_key, cluster_analysis):
2 """Get detailed information about a specific cluster"""
3 for cluster in cluster_analysis['top_clusters']:
4 if cluster['cluster_key'] == cluster_key:
5 return {
6 'size': cluster['size'],
7 'keywords': cluster['keywords'],
8 'topics': cluster['topics']
9 }
10 return Nonenumpy: Numerical computationsscikit-learn: Machine learning utilitiesminisom: Self-Organizing Map implementationmatplotlib: Static plottingplotly: Interactive visualizationspandas: Data manipulation