%%capture !python3 -m pip install nltk sacremoses pandas regex mock transformers>=4.33.2 mosestokenizer !python3 -c "import nltk; nltk.download('punkt')" !python3 -m pip install bitsandbytes scipy accelerate datasets !python3 -m pip install sentencepiece %cd IndicTransToolkit !python3 -m pip install --editable ./ %cd ..

import torch from transformers import AutoModelForSeq2SeqLM, BitsAndBytesConfig, AutoTokenizer

BATCH_SIZE = 4 DEVICE = "cuda" if torch.cuda.is_available() else "cpu" quantization = None

🔹 Uninstall Any Corrupted Versions

!pip uninstall -y torch torchvision torchaudio

🔹 Reinstall the Correct Version (CUDA 12.1 for GPU)

!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121

🔹 Restart Runtime (Run this manually if needed)

import os os._exit(00)

🔹 Verify Installation

import torch print(torch.version) print(torch.cuda.is_available()) # Should return True if using GPU

🔹 Run Your Code

import torch from transformers import AutoModelForSeq2SeqLM, BitsAndBytesConfig, AutoTokenizer from IndicTransToolkit import IndicProcessor

BATCH_SIZE = 4

def initialize_model_and_tokenizer(ckpt_dir, quantization): if quantization == "4-bit": qconfig = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, ) elif quantization == "8-bit": qconfig = BitsAndBytesConfig( load_in_8bit=True, bnb_8bit_use_double_quant=True, bnb_8bit_compute_dtype=torch.bfloat16, ) else: qconfig = None

tokenizer = AutoTokenizer.from_pretrained(ckpt_dir, trust_remote_code=True)
model = AutoModelForSeq2SeqLM.from_pretrained(
    ckpt_dir,
    trust_remote_code=True,
    low_cpu_mem_usage=True,
    quantization_config=qconfig,
)

if qconfig == None:
    model = model.to(DEVICE)
    if DEVICE == "cuda":
        model.half()

model.eval()

return tokenizer, model

def batch_translate(input_sentences, src_lang, tgt_lang, model, tokenizer, ip): translations = [] for i in range(0, len(input_sentences), BATCH_SIZE): batch = input_sentences[i : i + BATCH_SIZE]

    # Preprocess the batch and extract entity mappings
    batch = ip.preprocess_batch(batch, src_lang=src_lang, tgt_lang=tgt_lang)

    # Tokenize the batch and generate input encodings
    inputs = tokenizer(
        batch,
        truncation=True,
        padding="longest",
        return_tensors="pt",
        return_attention_mask=True,
    ).to(DEVICE)

    # Generate translations using the model
    with torch.no_grad():
        generated_tokens = model.generate(
            **inputs,
            use_cache=True,
            min_length=0,
            max_length=256,
            num_beams=5,
            num_return_sequences=1,
        )

    # Decode the generated tokens into text

    with tokenizer.as_target_tokenizer():
        generated_tokens = tokenizer.batch_decode(
            generated_tokens.detach().cpu().tolist(),
            skip_special_tokens=True,
            clean_up_tokenization_spaces=True,
        )

    # Postprocess the translations, including entity replacement
    translations += ip.postprocess_batch(generated_tokens, lang=tgt_lang)

    del inputs
    torch.cuda.empty_cache()

return translations

import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # Changed AutoTokenizery to AutoTokenizer

model_name = "ai4bharat/indictrans2-en-indic-1B" en_indic_model = AutoModelForSeq2SeqLM.from_pretrained(model_name, trust_remote_code=True) # Added trust_remote_code=True en_indic_tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) # Added trust_remote_code=True

device = "cuda" if torch.cuda.is_available() else "cpu" en_indic_model.to(device)

def batch_translate(sentences, src_lang, tgt_lang, model, tokenizer, processor): processed_inputs = processor.preprocess_batch(sentences, src_lang=src_lang, tgt_lang=tgt_lang) tokenized_inputs = tokenizer(processed_inputs, return_tensors="pt", padding=True).to(device) with torch.no_grad(): output_tokens = model.generate(**tokenized_inputs, max_length=128) return tokenizer.batch_decode(output_tokens, skip_special_tokens=True)

en_sents = ["Hello, how are you?", "This is a test sentence."] src_lang, tgt_lang = "eng_Latn", "hin_Deva"

pip install datasets

✅ Define Colloquial Hindi-English Data

colloquial_data = { "hi_input": [ "भाई, क्या हाल है?", "अरे यार, एकदम मस्त!", "चल भाई, कल मिलते हैं।", "भाई, बहुत बढ़िया लग रहा है!", "कोई चिंता नहीं, आराम से रहो भाई!" ], "en_target": [ "Bro, what's up?", "Hey dude, feeling awesome!", "Alright bro, see you tomorrow.", "Bro, this looks amazing!", "No worries, just chill bro!" ], "en_input": [ "What's up, bro?", "This party is so cool!", "See you tomorrow, man!", "Chill, dude, everything's fine!", "No worries, bro, it's all good!" ], "hi_target": [ "क्या हाल है भाई?", "यह पार्टी बहुत जबरदस्त है!", "कल मिलते हैं दोस्त!", "आराम से रहो, सब ठीक है!", "कोई चिंता नहीं भाई, सब बढ़िया!" ] } en_sents = [ "When I was young, I used to go to the park every day.", "He has many old books, which he inherited from his ancestors.", "I can't figure out how to solve my problem.", "She is very hardworking and intelligent, which is why she got all the good marks.", "We watched a new movie last week, which was very inspiring.", "If you had met me at that time, we would have gone out to eat.", "She went to the market with her sister to buy a new sari.", "Raj told me that he is going to his grandmother's house next month.", "All the kids were having fun at the party and were eating lots of sweets.", "My friend has invited me to his birthday party, and I will give him a gift.", ]

✅ Convert to Hugging Face Dataset Format

from datasets import Dataset, DatasetDict

dataset = Dataset.from_dict(colloquial_data) dataset = dataset.train_test_split(test_size=0.2) # 80% Train, 20% Test dataset = DatasetDict({"train": dataset["train"], "test": dataset["test"]})

diyamanna/bhashinillm