This is an INT4 quantized version of Google's Gemma-3 270M instruction-tuned model, optimized for Android deployment.
1import onnxruntime_genai as og
2import numpy as np
3
4# Load model
5model = og.Model("./gemma-3-270m-int4-genai")
6tokenizer = og.Tokenizer(model)
7
8# Generate text
9prompt = "Hello, how are you?"
10input_ids = tokenizer.encode(prompt)
11input_tokens = np.array([input_ids], dtype=np.int32)
12
13params = og.GeneratorParams(model)
14params.set_search_options(max_length=50)
15
16generator = og.Generator(model, params)
17generator.append_tokens(input_tokens)
18
19# Generate
20while not generator.is_done():
21 generator.generate_next_token()
22
23# Get output
24sequence = generator.get_sequence(0)
25output = tokenizer.decode(sequence)
26print(output)
1// Use provider override for Android compatibility
2OgaConfig* config = OgaCreateConfig(model_path);
3OgaConfigClearProviders(config);
4OgaConfigAppendProvider(config, "nnapi");
5OgaConfigAppendProvider(config, "xnnpack");
6OgaConfigAppendProvider(config, "cpu");
7OgaModel* model = OgaCreateModelFromConfig(config);
8
9// Use TokenizerStream for proper UTF-8 decoding
10OgaTokenizer* tokenizer = OgaCreateTokenizer(model);
11OgaTokenizerStream* stream = OgaCreateTokenizerStream(tokenizer);
12
13// After each generate_next_token:
14const char* text = OgaTokenizerStreamDecode(stream, new_token);
15// Emit 'text' to UI (readable UTF-8)
