┌─────────────────────────────────────────────────┐
│ Janus-Pro-7B Architecture │
├─────────────────────────────────────────────────┤
│ │
│ Input Text ──→ Tokenizer ──→ ┐ │
│ ├──→ DeepSeek-LLM │
│ Input Image ──→ SigLIP ──→ ┘ (7B, 30 layers│
│ 4096-dim) │
│ │
│ DeepSeek-LLM ──→ gen_head ──→ VQ Logits │
│ (4096→16384) │
│ │
│ VQ Tokens ──→ VQ-16 Decoder ──→ Output Image │
│ (16384 codebook, (384×384) │
│ 576 tokens/img) │
└─────────────────────────────────────────────────┘1 # Install Janus library
2 git clone https://github.com/deepseek-ai/Janus.git
3 cd Janus && pip install -e .
4
5 # Install other dependencies
6 pip install torch transformers Pillow numpy
1 import torch
2 import numpy as np
3 import PIL . Image
4 from transformers import AutoModelForCausalLM
5 from janus . models import MultiModalityCausalLM , VLChatProcessor
6
7 model_path = "asats/thumbnail-vlm-janus-pro"
8 processor = VLChatProcessor . from_pretrained ( model_path )
9 model = AutoModelForCausalLM . from_pretrained (
10 model_path , trust_remote_code = True , torch_dtype = torch . bfloat16
11 ) . cuda ( ) . eval ( )
12
13 # Generate thumbnail
14 prompt = "Professional tech review thumbnail: iPhone 16 with dramatic lighting, text 'BEST PHONE 2025'"
15 conversation = [
16 { "role" : "<|User|>" , "content" : prompt } ,
17 { "role" : "<|Assistant|>" , "content" : "" } ,
18 ]
19 sft_format = processor . apply_sft_template_for_multi_turn_prompts (
20 conversations = conversation , sft_format = processor . sft_format , system_prompt = ""
21 )
22 prompt_text = sft_format + processor . image_start_tag
23
24 with torch . inference_mode ( ) :
25 input_ids = torch . LongTensor ( processor . tokenizer . encode ( prompt_text ) )
26 tokens = torch . zeros ( ( 2 , len ( input_ids ) ) , dtype = torch . int ) . cuda ( )
27 tokens [ 0 ] = input_ids # conditional
28 tokens [ 1 ] = input_ids ; tokens [ 1 , 1 : - 1 ] = processor . pad_id # unconditional
29
30 inputs_embeds = model . language_model . get_input_embeddings ( ) ( tokens )
31 generated = torch . zeros ( ( 1 , 576 ) , dtype = torch . int ) . cuda ( )
32
33 past_kv = None
34 for t in range ( 576 ) :
35 outputs = model . language_model . model ( inputs_embeds = inputs_embeds , use_cache = True , past_key_values = past_kv )
36 past_kv = outputs . past_key_values
37 logits = model . gen_head ( outputs . last_hidden_state [ : , - 1 , : ] )
38 guided = logits [ 1 : 2 ] + 5.0 * ( logits [ 0 : 1 ] - logits [ 1 : 2 ] )
39 next_tok = torch . multinomial ( torch . softmax ( guided , - 1 ) , 1 )
40 generated [ : , t ] = next_tok . squeeze ( - 1 )
41 img_emb = model . prepare_gen_img_embeds ( torch . cat ( [ next_tok , next_tok ] , 0 ) . squeeze ( - 1 ) )
42 inputs_embeds = img_emb . unsqueeze ( 1 )
43
44 dec = model . gen_vision_model . decode_code ( generated , shape = [ 1 , 8 , 24 , 24 ] )
45 img = np . clip ( ( dec . float ( ) . cpu ( ) . numpy ( ) . transpose ( 0 , 2 , 3 , 1 ) + 1 ) / 2 * 255 , 0 , 255 ) . astype ( np . uint8 )
46 PIL . Image . fromarray ( img [ 0 ] ) . save ( "thumbnail.png" )
1 # Uses model's understanding to caption, then generates
2 python scripts / inference_janus . py - - mode image - - input_image photo . jpg
1 # Uses both text instruction and reference image
2 python scripts / inference_janus . py - - mode both \
3 - - prompt "Create a cooking video thumbnail with text 'EASY RECIPE'" \
4 - - input_image food_photo . jpg
1 # Launch via HF Jobs API
2 from huggingface_hub import HfApi
3 api = HfApi ( )
4
5 # Requires: a100-large hardware, 8h timeout
6 # Dependencies: torch, transformers, datasets, Pillow, numpy, tqdm,
7 # trackio, accelerate, janus @ git+https://github.com/deepseek-ai/Janus.git
1 # Clone repo and install
2 git clone https://github.com/deepseek-ai/Janus.git && cd Janus && pip install -e .
3 pip install torch transformers datasets Pillow numpy tqdm trackio accelerate
4
5 # Run training (needs ~40GB VRAM, A100 recommended)
6 python run_training.py
For a lighter approach using OmniGen-v1 (3.8B params, LoRA fine-tuning on single 24GB GPU):
1 pip install OmniGen accelerate peft
2 accelerate launch train_omnigen.py \
3 --model_name_or_path Shitao/OmniGen-v1 \
4 --json_file train.jsonl \
5 --image_path ./images \
6 --use_lora --lora_rank 8 \
7 --lr 1e-3 --epochs 3
├── README.md # This file
├── scripts/
│ ├── run_training.py # End-to-end training pipeline (data prep + train + eval)
│ ├── inference_janus.py # Inference for all 3 input modes
│ ├── train_janus.py # Modular Janus training script
│ ├── train_omnigen.py # Alternative OmniGen LoRA training
│ └── prepare_data.py # Data preparation utilities
