Views
No views yet
1# Clone and setup
2cd Zenith/V1/7B
3pip install -r requirements.txt1# Full fine-tuning
2python train.py \
3 --base_model Qwen/Qwen2.5-Coder-7B \
4 --train_data path/to/train.json \
5 --epochs 3 \
6 --batch_size 4 \
7 --learning_rate 2e-5
8
9# LoRA fine-tuning (recommended for most users)
10python train.py \
11 --base_model Qwen/Qwen2.5-Coder-7B \
12 --train_data path/to/train.json \
13 --use_lora \
14 --lora_r 16 \
15 --lora_alpha 32 \
16 --epochs 3 \
17 --batch_size 81# Interactive mode
2python inference.py --checkpoint ./outputs/checkpoint-final
3
4# Single prompt
5python inference.py \
6 --checkpoint ./outputs/checkpoint-final \
7 --prompt "Write a Python function to reverse a linked list" \
8 --max_new_tokens 5121# Build and run with Ollama
2ollama create zenith-7b -f Modelfile
3ollama run zenith-7b "Explain quantum computing in simple terms"Zenith/V1/7B/
├── configs/ # Configuration files
│ ├── zenith_config.py # Model architecture config
│ ├── data_config.py # Data processing config
│ └── training_config.py # Training hyperparameters
├── data/ # Data processing modules
│ ├── openthoughts_processor.py
│ ├── quality_filter.py
│ ├── curriculum_sampler.py
│ ├── advanced_tokenizer.py
│ └── preprocessing.py
├── src/ # Source code
│ ├── models/
│ │ ├── zenith_model.py
│ │ ├── dense_layer.py
│ │ └── moe_layer.py
│ └── utils/
├── scripts/ # Utility scripts
├── tests/ # Test suite
├── train.py # Main training script
├── inference.py # Inference and generation
├── test_model.py # Model validation tests
├── finetune_qwen.py # Qwen fine-tuning guide
├── Modelfile # Ollama configuration
├── requirements.txt # Python dependencies
└── README.md # This fileconfigs/zenith_config.py:1from configs.zenith_config import get_7b_config
2
3config = get_7b_config()
4# Parameters:
5# - hidden_size: 4096
6# - num_layers: 32
7# - num_heads: 32
8# - num_experts: 0 (dense only, set >1 for MoE)
9# - use_eq_adapter: True (emotional intelligence)
10# - max_seq_len: 81921from data.openthoughts_processor import OpenThoughtsProcessor, OpenThoughtsConfig
2
3config = OpenThoughtsConfig(
4 dataset_name="open-thoughts/OpenThoughts3-1.2M",
5 streaming=True,
6 quality_filtering=True,
7 curriculum_learning=True,
8 augmentation=True
9)
10processor = OpenThoughtsProcessor(config)
11dataset = processor.load_dataset()python train.py --use_moe --num_experts 8python train.py --use_eq_adapter --eq_loss_weight 0.11# LoRA
2python train.py --use_lora --lora_r 16 --lora_alpha 32
3
4# QLoRA (4-bit quantization)
5python train.py --use_qlora --use_lora --lora_r 8python test_model.pyrequirements.txt for full dependencies. Key packages:--mixed_precision bf16 for faster training (Ampere+ GPUs)--max_seq_length1@misc{zenith-7b-2025,
2 title={Zenith-7B: A Hybrid MoE Model for Code and Emotional Intelligence},
3 year={2025},
4 publisher={Zenith Project}
5}