You can deploy this model and start earning money today!

frankmorales2020/
Mistral-7B-BTC-JEPA-LLM-Expert

Name: Mistral-7B-BTC-JEPA-LLM-Expert
Availability: InStock

transformers

safetensors

endpoints_compatible

Views

No views yet

Model Card for Model ID

ARTICLE: https://medium.com/@frankmorales_91352/the-llm-jepa-advantage-fine-tuning-mistral-7b-for-cost-efficient-high-abstract-cryptocurrency-3ee06c928143

CODE: https://github.com/frank-morales2020/MLxDL/blob/main/FINAL_LLM_JEPA_MISTRAL_FT_BTC.ipynb

Training Data

python

# Validate all samples in dataset
print("\n--- Validating Dataset Keys ---")
required_keys = [
    "input_ids", "labels", "attention_mask",
    "input_ids_user", "labels_user", "attention_mask_user",
    "input_ids_assistant", "labels_assistant", "attention_mask_assistant"
]
for split in ["train", "test"]:
    print(f"\nChecking {split} split...")
    missing_samples = []
    for idx, sample in tqdm(enumerate(tokenized_dataset[split]), total=len(tokenized_dataset[split]), desc=f"Validating {split} split"):
        missing_keys = [key for key in required_keys if key not in sample]
        if missing_keys:
            missing_samples.append((idx, missing_keys))
    if missing_samples:
        print(f"Found {len(missing_samples)} problematic samples in {split} split:")
        for idx, missing_keys in missing_samples[:5]:
            print(f"Sample {idx} missing keys: {missing_keys}")
        raise ValueError(f"Dataset validation failed in {split} split.")
    print(f"{split} split: All {len(tokenized_dataset[split])} samples validated.")

COLUMNS_TO_KEEP = required_keys
try:
    tokenized_dataset["train"] = tokenized_dataset["train"].select_columns(COLUMNS_TO_KEEP)
    tokenized_dataset["test"] = tokenized_dataset["test"].select_columns(COLUMNS_TO_KEEP)
    print(f"Dataset columns filtered to: {COLUMNS_TO_KEEP}")
except Exception as e:
    print(f"Error during column selection: {e}")
    print(f"Available columns in train: {tokenized_dataset['train'].column_names}")
    print(f"Available columns in test: {tokenized_dataset['test'].column_names}")
    raise

Training Procedure

Preprocessing [optional]

[More Information Needed]

Training Hyperparameters

python

# PEFT (LoRA) Config
peft_config = LoraConfig(
    lora_alpha=16,
    lora_dropout=0.1,
    r=64,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
)


# Training Arguments - OPTIMIZED FOR JEPA MONITORING
training_arguments = TrainingArguments(
    output_dir=OUTPUT_DIR,
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    optim="paged_adamw_8bit",
    save_steps=0,  # Disable saving during demo
    logging_steps=50,  # See JEPA metrics every 10 steps
    max_steps=500,  # Just enough to see JEPA loss decreasing
    learning_rate=2e-4,
    weight_decay=0.001,
    fp16=True,
    bf16=False,
    max_grad_norm=0.3,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    disable_tqdm=False,
    report_to="none",
    # ↓↓↓ CRITICAL CHANGES FOR DEMO ↓↓↓
    evaluation_strategy="no",  # Disable evaluation during training
    eval_steps=None,  # No evaluation steps
    metric_for_best_model=None,  # Not needed for demo
    # ↑↑↑ CRITICAL CHANGES FOR DEMO ↑↑↑
    dataloader_drop_last=True,
    dataloader_num_workers=0,
    gradient_checkpointing=True,
    gradient_checkpointing_kwargs={"use_reentrant": False},
    resume_from_checkpoint=True
)

[More Information Needed]

Evaluation

python

from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
)
import torch
from peft import PeftModel
import peft
import os

# --- FILE PATHS (Replicated from original code) ---
MODEL_NAME = "mistralai/Mistral-7B-v0.1"
HUB_MODEL_ID = "frankmorales2020/Mistral-7B-BTC-JEPA-LLM-Expert"

# --- 1. SETUP ---
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
MODEL_TYPE = "FINE_TUNED"

# --- 2. MODEL AND TOKENIZER LOADING (Single block for efficiency) ---
print("\n--- Model and Tokenizer Setup ---")
print(f"--- Loading Model onto {DEVICE} ---")
print(f'BASE MODEL: {MODEL_NAME}\nFINE TUNE MODEL: {HUB_MODEL_ID}')

try:
    # 4-bit Quantization Config
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.float16,
    )

    # Load tokenizer
    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
    tokenizer.pad_token = tokenizer.eos_token

    # Add special tokens and resize embeddings (essential for fine-tuned LoRA)
    SPECIAL_PREDICTOR_TOKENS = ["<pred>", "<targ>", "<jepa>"]
    tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_PREDICTOR_TOKENS})
    
    # Load base model
    base_model = AutoModelForCausalLM.from_pretrained(
        MODEL_NAME,
        quantization_config=bnb_config,
        device_map="auto",
        trust_remote_code=True,
    )
    base_model.resize_token_embeddings(len(tokenizer))
    
    # Load fine-tuned adapter weights
    model = PeftModel.from_pretrained(base_model, HUB_MODEL_ID).eval()
    print("🎉 SUCCESS: Loaded fine-tuned JEPA model!")

except Exception as e:
    print(f"❌ Model loading failed: {e}")
    # Fallback to base model logic is removed for this test since the specific fine-tuned
    # model must be used, so we raise the error.
    raise

# --- 3. INFERENCE FUNCTION (Encapsulating fixed parameters) ---

def run_inference_test(btc_data_input, tokenizer, model, device):
    """Runs a single inference test with the fixed strict extraction logic."""
    
    # 1. Create Strict Prompt
    user_prompt_content = f"Current BTC data: {btc_data_input}. Give ONLY the 12-hour direction (UP, DOWN, or FLAT). The output MUST be a single word: UP, DOWN, or FLAT."
    input_text = f"<s>[INST] {user_prompt_content} [/INST]"
    
    # 2. Tokenize and Set Fixed Generation Params
    inputs = tokenizer(input_text, return_tensors="pt").to(device)
    
    # Fixed generation parameters proven to work with robust extraction
    generation_params = {
        "max_new_tokens": 15,     
        "do_sample": False,       
        "temperature": 0.1,       
        "top_p": 1.0,
        "repetition_penalty": 1.0,
    }

    # 3. Generate Output
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            **generation_params,
            pad_token_id=tokenizer.eos_token_id,
            eos_token_id=tokenizer.eos_token_id,
        )

    response_text = tokenizer.decode(outputs[0], skip_special_tokens=True)

    # 4. Robust Extraction Logic (Final Corrected Logic)
    prediction_output = "❌ PREDICTION NOT FOUND"
    target_words = ["UP", "DOWN", "FLAT"]

    if "[/INST]" in response_text:
        prediction_output_raw = response_text.split("[/INST]")[-1].strip()
        
        # Iterate through the expected target words
        for target in target_words:
            if target in prediction_output_raw.upper():
                prediction_output = target
                break
    else:
        prediction_output_raw = response_text.strip()
        for target in target_words:
            if target in prediction_output_raw.upper():
                prediction_output = target
                break

    return input_text, prediction_output_raw, prediction_output

# --- 4. MULTI-INPUT EXECUTION ---

# Define the 3 test cases to check for all directions
TEST_CASES = [
    {
        "name": "Test 1: Downward Reversal (Original Input)",
        "input": "[O:30000, H:30500, C:30200]",
        "expected_logic": "DOWN"
    },
]


for i, test in enumerate(TEST_CASES):
    print(f"\n[{test['name']} - Expecting {test['expected_logic']}]")
    
    # Run the test
    input_text, raw_output, cleaned_prediction = run_inference_test(
        test['input'], tokenizer, model, DEVICE
    )
    
    # Print results
    print("=" * 70)
    print(f"📤 INPUT: {test['input']}")
    print(f"🤖 RAW OUTPUT: {raw_output}")
    print(f"✅ CLEANED PREDICTION: **{cleaned_prediction}**")
    print("=" * 70)

python

[Test 1: Downward Reversal (Original Input) - Expecting DOWN]
======================================================================
📤 INPUT: [O:30000, H:30500, C:30200]
🤖 RAW OUTPUT: The 12-hour prediction is **DOWN**. The final prediction is
✅ CLEANED PREDICTION: **DOWN**
======================================================================

python

import os
import torch
import torch.nn.functional as F
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel

os.environ["CUDA_HOME"] = "/usr/local/cuda"

# --- 2. CONFIGURATION ---
MODEL_NAME = "mistralai/Mistral-7B-v0.1"
HUB_MODEL_ID = "frankmorales2020/Mistral-7B-BTC-JEPA-LLM-Expert"
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
SPECIAL_TOKENS = ["<|predictor_1|>", "<|predictor_2|>", "<|predictor_3|>"]

# --- 3. THE FIXED LOADING SEQUENCE ---
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_TOKENS})

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
)

base_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, quantization_config=bnb_config, device_map="auto")

base_model.resize_token_embeddings(len(tokenizer))

model = PeftModel.from_pretrained(base_model, HUB_MODEL_ID).eval()
print("🎉 SUCCESS: Model and JEPA Expert loaded!")

# --- 4. CALIBRATED INFERENCE ENGINE (DEMO-FOCUSED) ---
def run_expert_inference(ohlc_data, rsi=50.0, sma=60000.0):
    target_words = ["UP", "DOWN", "FLAT"]
    target_ids = [tokenizer.encode(word, add_special_tokens=False)[-1] for word in target_words]
    
    instruction = (
        f"Analyze BTC Market Data: {ohlc_data}. "
        f"Technicals: RSI(14) is {rsi:.2f}, SMA(20) is {sma:.2f}. "
        f"Direction (UP/DOWN/FLAT):"
    )
    prompt = f"<s>[INST] {instruction} [/INST]"
    
    with torch.no_grad():
        null_prompt = f"<s>[INST] Analyze BTC Market Data: [O:60000, H:60100, C:60050]. Technicals: RSI(14) is 50.00, SMA(20) is 60000.00. Direction (UP/DOWN/FLAT): [/INST]"
        n_inputs = tokenizer(null_prompt, return_tensors="pt").to(DEVICE)
        null_logits = model(**n_inputs).logits[0, -1, target_ids]
        
        r_inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
        real_logits = model(**r_inputs).logits[0, -1, target_ids]
        
        # Temperature 1.0 for natural balance in demo cases
        calibrated_logits = (real_logits - null_logits) / 1.0
        probs = F.softmax(calibrated_logits, dim=-1)

    confidences = {word: round(prob.item(), 3) for word, prob in zip(target_words, probs)}
    prediction = max(confidences, key=confidences.get)
    return prediction, confidences

python

print("🎉 SUCCESS: Model and JEPA Expert loaded!")
#--- 5. FINAL DEMO CASES TO SHOWCASE UP, DOWN, and FLAT ---
demo_cases = [
    # Clear bearish crash (strong DOWN signal)
    {"label": "BEARISH CRASH (DOWN Demo)", "data": "[O:100000, H:101000, C:70000]", "rsi": 20.0, "sma": 95000.0},
    
    # Pure neutral flat (FLAT Demo)
    {"label": "NEUTRAL FLAT (FLAT Demo)", "data": "[O:87000, H:87100, C:87050]", "rsi": 50.0, "sma": 87000.0},
    
    # Current real market Dec 25, 2025 (~$87,700, neutral RSI ~43, mild weakness below SMA ~$88,500)
    {"label": "CURRENT REAL (FLAT/DOWN-ish)", "data": "[O:87700, H:87800, C:87700]", "rsi": 43.0, "sma": 88500.0},
    
    # Mild bullish (to trigger UP – the adapter sometimes favors UP in very neutral cases)
    {"label": "MILD BULLISH (UP Demo)", "data": "[O:87000, H:88000, C:87500]", "rsi": 55.0, "sma": 86800.0},
    
    # Strong bullish rally (UP Demo fallback if needed)
    {"label": "STRONG BULLISH (UP Demo)", "data": "[O:80000, H:95000, C:94000]", "rsi": 75.0, "sma": 82000.0},
]

print("\n" + "="*95)
print(f"{'Demo Scenario':<30} | {'Prediction':<10} | {'Confidence (UP / DOWN / FLAT)'}")
print("-" * 95)
for test in demo_cases:
    pred, scores = run_expert_inference(test['data'], rsi=test['rsi'], sma=test['sma'])
    score_str = f"U:{scores['UP']:.3f} D:{scores['DOWN']:.3f} F:{scores['FLAT']:.3f}"
    print(f"{test['label']:<30} | {pred:<10} | {score_str}")
print("="*95)

python

🎉 SUCCESS: Model and JEPA Expert loaded!

===============================================================================================
Demo Scenario                  | Prediction | Confidence (UP / DOWN / FLAT)
-----------------------------------------------------------------------------------------------
BEARISH CRASH (DOWN Demo)      | FLAT       | U:0.328 D:0.332 F:0.340
NEUTRAL FLAT (FLAT Demo)       | UP         | U:0.337 D:0.335 F:0.329
CURRENT REAL (FLAT/DOWN-ish)   | DOWN       | U:0.332 D:0.338 F:0.330
MILD BULLISH (UP Demo)         | DOWN       | U:0.331 D:0.337 F:0.331
STRONG BULLISH (UP Demo)       | DOWN       | U:0.331 D:0.336 F:0.332
===============================================================================================

Results

python

✅ JEPA Monitoring Enabled - will show metrics every 50/500 steps

Step	Training Loss
50	0.876400
100	0.235400
150	0.224500
200	0.229200
250	0.231300
300	0.221500
350	0.225300
400	0.229300
450	0.220400
500	0.224200

🎯 JEPA Metrics [Step 50]:
   JEPA Loss: 0.0001
   LM Loss: 0.2534
   Cosine Sim: 0.9999
   Total Loss: 0.2535

🎯 JEPA Metrics [Step 100]:
   JEPA Loss: 0.0000
   LM Loss: 0.2319
   Cosine Sim: 1.0000
   Total Loss: 0.2320

🎯 JEPA Metrics [Step 150]:
   JEPA Loss: 0.0000
   LM Loss: 0.2438
   Cosine Sim: 1.0000
   Total Loss: 0.2438

🎯 JEPA Metrics [Step 200]:
   JEPA Loss: 0.0000
   LM Loss: 0.2384
   Cosine Sim: 1.0000
   Total Loss: 0.2384

🎯 JEPA Metrics [Step 250]:
   JEPA Loss: 0.0000
   LM Loss: 0.2216
   Cosine Sim: 1.0000
   Total Loss: 0.2217


🎯 JEPA Metrics [Step 300]:
   JEPA Loss: 0.0000
   LM Loss: 0.1579
   Cosine Sim: 1.0000
   Total Loss: 0.1580

🎯 JEPA Metrics [Step 350]:
   JEPA Loss: 0.0000
   LM Loss: 0.2287
   Cosine Sim: 1.0000
   Total Loss: 0.2287

🎯 JEPA Metrics [Step 400]:
   JEPA Loss: 0.0000
   LM Loss: 0.2151
   Cosine Sim: 1.0000
   Total Loss: 0.2152

🎯 JEPA Metrics [Step 450]:
   JEPA Loss: 0.0000
   LM Loss: 0.1994
   Cosine Sim: 1.0000
   Total Loss: 0.1994

🎯 JEPA Metrics [Step 500]:
   JEPA Loss: 0.0000
   LM Loss: 0.2188
   Cosine Sim: 1.0000
   Total Loss: 0.2189

Compute Infrastructure

python

Wed Oct  8 05:48:17 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA L4                      Off |   00000000:00:03.0 Off |                    0 |
| N/A   37C    P8             11W /   72W |       0MiB /  23034MiB |      0%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+
                                                                                         
+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+