You can deploy this model and start earning money today!

LeeJiaMing/
Gemma_3_270m_commonsenseQA_tunned

Name: Gemma_3_270m_commonsenseQA_tunned
Availability: InStock

safetensors

Views

No views yet

GEmma_3_270m Fine-Tuned on Common Sense QA

This repository contains a memory-optimized fine-tuned version of GEmma_3_270m for Common Sense Question Answering (CSQA). The model is trained to perform CSQA tasks efficiently while minimizing GPU memory usage. Training, evaluation, and inference are all designed to be memory-safe.

Model Overview

Base Model: google/gemma-3-270m
Task: Common Sense Question Answering (CSQA)
Fine-tuning: Memory-efficient training with gradient checkpointing, mixed precision (FP16), and small batch sizes.
Dataset: Custom CSQA dataset, up to 20k samples.
Tokenizer: Hugging Face AutoTokenizer (adds padding token if missing).
Output: Fine-tuned model saved at ./gemma3-finetuned.

Features

Memory-efficient training and evaluation.
Logging of CPU, GPU, and memory metrics during training.
Separate standalone evaluation to avoid OOM.
Checkpointing for safe resuming of interrupted training.
Latency metrics: Time to first token (TTFT), total generation time, average generated tokens.

Installation

bash

pip install torch transformers datasets evaluate nltk psutil pynvml pandas tqdm scikit-learn



# ==============================================================================
# 1. Setup and Memory Optimization
# This command must be run in the shell/environment *before* starting the script
# for it to take effect, but it's included here for documentation.
# If running this inside a Jupyter/Colab notebook, use: %env PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
# ==============================================================================
# export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128

# ==============================================================================
# 2. Download Required NLTK Data
# ==============================================================================
import nltk
try:
    nltk.data.find('tokenizers/punkt')
except nltk.downloader.DownloadError:
    print("Downloading 'punkt' NLTK data...")
    nltk.download('punkt')
try:
    nltk.data.find('corpora/wordnet')
except nltk.downloader.DownloadError:
    print("Downloading 'wordnet' NLTK data...")
    nltk.download('wordnet')

# ==============================================================================
# 3. Model Loading (Requires model files in './gemma3-finetuned')
# ==============================================================================
from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, DataCollatorForLanguageModeling
import torch
import psutil
try:
    import pynvml
except ImportError:
    print("Warning: pynvml not found. GPU metrics will be skipped. Install with: pip install pynvml")
    pynvml = None

# --- Configuration ---
model_path = "./gemma3-finetuned"
# --- End Configuration ---

print(f"Loading model from {model_path}...")
try:
    tokenizer = AutoTokenizer.from_pretrained(model_path)
    # Using device_map="auto" and torch_dtype="auto" for memory-efficient loading
    model = AutoModelForCausalLM.from_pretrained(
        model_path,
        device_map="auto",
        torch_dtype="auto"
    )
    print("Model loaded successfully.")
except Exception as e:
    print(f"Error loading model: {e}")
    # Exit or mock model for demonstration if loading fails
    # raise e

# ==============================================================================
# 4. Running Inference
# ==============================================================================
print("\n" + "="*50)
print("Starting Inference Example")
print("="*50)

query = "Why do people put ice in drinks?"
prompt = f"### Query:\n{query}\n\n### Answer:\n"
print(f"Prompt: {prompt.strip()}")

if 'model' in locals():
    # Ensure model and inputs are on the same device
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    # Note: pad_token_id and eos_token_id are often the same for Causal LMs like Gemma
    outputs = model.generate(
        **inputs,
        max_new_tokens=128,
        do_sample=False,
        pad_token_id=tokenizer.pad_token_id,
        eos_token_id=tokenizer.eos_token_id
    )

    # Decode only the newly generated tokens
    answer = tokenizer.decode(outputs[0][inputs['input_ids'].shape[-1]:], skip_special_tokens=True)
    print("\nAnswer:", answer)
else:
    print("Skipping inference: Model not loaded.")

# ==============================================================================
# 5. Metrics & System Information
# ==============================================================================
print("\n" + "="*50)
print("Collecting System Metrics")
print("="*50)

def get_system_metrics():
    metrics = {}
    if torch.cuda.is_available() and pynvml:
        try:
            pynvml.nvmlInit()
            handle = pynvml.nvmlDeviceGetHandleByIndex(0)
            mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
            util = pynvml.nvmlDeviceGetUtilizationRates(handle)
            metrics.update({
                "gpu_memory_used_GB": round(mem_info.used / 1e9, 2),
                "gpu_memory_total_GB": round(mem_info.total / 1e9, 2),
                "gpu_utilization_%": util.gpu
            })
            pynvml.nvmlShutdown()
        except Exception as e:
            metrics["gpu_error"] = str(e)
            print(f"Error collecting GPU metrics: {e}")
            if pynvml: pynvml.nvmlShutdown()

    metrics.update({
        "cpu_usage_%": psutil.cpu_percent(interval=0.1),
        "memory_usage_%": psutil.virtual_memory().percent
    })
    return metrics

system_metrics = get_system_metrics()
for key, value in system_metrics.items():
    print(f"{key}: {value}")


# ==============================================================================
# 6. Standalone Evaluation (Requires 'eval_dataset')
# Evaluation is decoupled from training to prevent OOM.
# ==============================================================================
print("\n" + "="*50)
print("Starting Standalone Evaluation (Requires 'eval_dataset')")
print("="*50)

def run_standalone_evaluation(model, tokenizer, eval_dataset):
    model.eval()
    # Limiting evaluation to a subset (e.g., first 50 samples) to save time and memory
    eval_subset = eval_dataset.select(range(min(50, len(eval_dataset))))
    total_loss = 0
    for i, sample in enumerate(eval_subset):
        if i % 10 == 0:
             print(f"Evaluating sample {i}/{len(eval_subset)}...")
        try:
            with torch.no_grad():
                # Assuming 'eval_dataset' provides 'input_ids', 'attention_mask', 'labels'
                inputs = {
                    'input_ids': torch.tensor(sample['input_ids']).unsqueeze(0).to(model.device),
                    'attention_mask': torch.tensor(sample['attention_mask']).unsqueeze(0).to(model.device),
                    'labels': torch.tensor(sample['labels']).unsqueeze(0).to(model.device)
                }
                outputs = model(**inputs)
                total_loss += outputs.loss.item()
        except Exception as e:
            print(f"Skipping sample {i} due to error: {e}")
            continue

    if len(eval_subset) > 0:
        avg_loss = total_loss / len(eval_subset)
        perplexity = torch.exp(torch.tensor(avg_loss)).item()
        return {"eval_loss": avg_loss, "perplexity": perplexity}
    else:
        return {"eval_loss": 0, "perplexity": 0}

# --- Placeholder for loading datasets ---
# NOTE: 'eval_dataset' and 'train_dataset' must be defined and loaded
# using the Hugging Face 'datasets' library before this section can run.
# Example: from datasets import load_dataset; dataset = load_dataset('some_data'); eval_dataset = dataset['test']
if 'eval_dataset' in locals() and 'model' in locals():
    eval_results = run_standalone_evaluation(model, tokenizer, eval_dataset)
    print("\nEvaluation Results:", eval_results)
else:
    print("Skipping evaluation: 'eval_dataset' or 'model' not defined/loaded.")


# ==============================================================================
# 7. Training (Requires 'train_dataset')
# This section sets up the Trainer but does not execute training unless
# 'train_dataset' is available.
# ==============================================================================
print("\n" + "="*50)
print("Setting up Memory-Efficient Training (Requires 'train_dataset')")
print("="*50)

# --- Training Configuration ---
training_output_dir = "./gemma3-finetuned"
training_checkpoints_dir = "./training_checkpoints"
# --- End Training Configuration ---

if 'train_dataset' in locals() and 'model' in locals():
    print("Initializing Trainer...")
    data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)

    training_args = TrainingArguments(
        output_dir=training_output_dir,
        overwrite_output_dir=True,
        per_device_train_batch_size=2,      # Low batch size to save memory
        gradient_accumulation_steps=8,      # Effective batch size is 16 (2 * 8)
        num_train_epochs=2,
        learning_rate=2e-5,
        weight_decay=0.01,
        logging_steps=100,
        save_strategy="steps",
        save_steps=200,
        save_total_limit=2,
        fp16=True,                          # Enables mixed precision
        dataloader_num_workers=0
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=None,                  # Evaluation is run standalone (Section 6)
        tokenizer=tokenizer,
        data_collator=data_collator
    )

    print("Trainer initialized. Starting training...")
    try:
        # Resume from a checkpoint if one is found/specified
        # Replace 'checkpoint-XXX' with the actual checkpoint directory name
        trainer.train(resume_from_checkpoint=f"{training_checkpoints_dir}/checkpoint-XXX")
        trainer.save_model(training_output_dir)
        print(f"Training complete. Final model saved to: {training_output_dir}")
    except FileNotFoundError:
        print("Starting training from scratch (no checkpoint found/specified)...")
        trainer.train()
        trainer.save_model(training_output_dir)
        print(f"Training complete. Final model saved to: {training_output_dir}")
    except Exception as e:
        print(f"An error occurred during training: {e}")
else:
    print("Skipping training setup: 'train_dataset' or 'model' not defined/loaded.")

# ==============================================================================
# 8. Logs & Checkpoints (Documented)
# ==============================================================================
# Training logs saved to: training_logs.csv
# Final metrics saved to: final_metrics.csv
# Checkpoints directory: ./training_checkpoints
# ==============================================================================

LeeJiaMing/Gemma_3_270m_commonsenseQA_tunned

GEmma_3_270m Fine-Tuned on Common Sense QA

Model Overview

Features

Installation

LeeJiaMing/
Gemma_3_270m_commonsenseQA_tunned