!pip install transformers datasets rouge-score torch accelerate bitsandbytes
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from datasets import load_dataset
from rouge_score import rouge_scorer
import pandas as pd
import numpy as np
from tqdm import tqdm
Load model and tokenizer
print("Loading model and tokenizer...")
tokenizer = AutoTokenizer.from_pretrained("beyzasezer/DeepseekR1-Distill-Llama-8B-unsloth-bnb-4bit_finetune")
model = AutoModelForCausalLM.from_pretrained(
"beyzasezer/DeepseekR1-Distill-Llama-8B-unsloth-bnb-4bit_finetune",
torch_dtype=torch.float16,
device_map="auto"
)
Load dataset
print("Loading dataset...")
dataset = load_dataset("Isotonic/marketing_email_samples", split="test")
dataset = dataset.select(range(30)) #################az row ile işlem yap satırı
# Generate text
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_length,
do_sample=True,
temperature=0.7,
top_p=0.9,
)
# Decode generated text
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
# Remove the prompt from the generated text
if generated_text.startswith(prompt):
generated_text = generated_text[len(prompt):].strip()
return generated_text
for i, item in enumerate(tqdm(dataset_sample)):
# Create prompt - adjusted for actual dataset columns
prompt = f"Product: {item['product']}\nDescription: {item['description']}\n\nWrite a marketing email:\n"
# Generate email
try:
generated_email = generate_email(prompt)
# Get reference email
reference_email = item['marketing_email']
# Calculate ROUGE scores
scores = scorer.score(reference_email, generated_email)
# Store scores
for metric, score_obj in scores.items():
all_scores[f'{metric}_precision'].append(score_obj.precision)
all_scores[f'{metric}_recall'].append(score_obj.recall)
all_scores[f'{metric}_fmeasure'].append(score_obj.fmeasure)
# Store individual results
results.append({
'id': i,
'prompt': prompt,
'reference': reference_email,
'generated': generated_email,
'scores': scores
})
except Exception as e:
print(f"Error processing item {i}: {e}")
Calculate average scores
average_scores = {}
for metric, values in all_scores.items():
average_scores[metric] = np.mean(values)
Print average scores
print("\nAverage ROUGE Scores:")
for metric, score in average_scores.items():
print(f"{metric}: {score:.4f}")
Save results to CSV
results_df = pd.DataFrame({
'id': [r['id'] for r in results],
'reference': [r['reference'] for r in results],
'generated': [r['generated'] for r in results],
'rouge1_f': [r['scores']['rouge1'].fmeasure for r in results],
'rouge2_f': [r['scores']['rouge2'].fmeasure for r in results],
'rougeL_f': [r['scores']['rougeL'].fmeasure for r in results]
})
results_df.to_csv('rouge_evaluation_results_deepseekFinetuned_30Sample.csv', index=False)
print("Results saved to rouge_evaluation_results_deepseekFinetuned_30Sample.csv")
prompt ="Generate a marketing email with the subject about wrist watches for mother's day campaign"
generated_email = generate_email(prompt)
generated_email
