Views
No views yet
pip install -U huggingface_hub["cli"]hf download savvadesogle/T-lite-it-2.1-int8-ov --local-dir ./T-lite-it-2.1-int8-ovoptimum-cli export openvino with the following parameters:1optimum-cli export openvino ^
2 --model ./T-lite-it-2.1 ^
3 --task text-generation-with-past ^
4 --weight-format int8 ^
5 ./T-lite-it-2.1-int8-ovset OPENARC_API_KEY=BIG_KEY
openarc serve start --host 127.0.0.1set OPENARC_API_KEY=BIG_KEY
openarc add --model-name T-lite-it-2.1 --model-path ./T-lite-it-2.1-int8-ov --engine ovgenai --model-type llm --device GPU
openarc load T-lite-it-2.1
openarc bench T-lite-it-2.1http://127.0.0.1:8000/v1/chat/completionscurl http://127.0.0.1:8000/v1/chat/completions ^
-H "Content-Type: application/json" ^
-H "Authorization: Bearer 234" ^
-d "{\"model\":\"T-lite-it-2.1-int8-ov\",\"messages\":[{\"role\":\"user\",\"content\":\"Расскажи анекдот\"}],\"temperature\":0.7,\"max_tokens\":128}"openarc bench T-lite-it-2.1-int8-ov2026-01-04 01:54:24,140 - INFO - [T-lite-it-2.1-int8-ov LLM Worker] Metrics: {'load_time (s)': 24.9, 'ttft (s)': 0.09, 'tpot (ms)': 33.26695, 'prefill_throughput (tokens/s)': 5889.65, 'decode_throughput (tokens/s)': 30.05987, 'decode_duration (s)': 4.31193, 'input_token': 512, 'new_token': 128, 'total_token': 640, 'stream': False}
2026-01-04 01:54:24,140 - INFO - [bench] model=T-lite-it-2.1-int8-ov input_ids_len=512 metrics={'load_time (s)': 24.9, 'ttft (s)': 0.09, 'tpot (ms)': 33.26695, 'prefill_throughput (tokens/s)': 5889.65, 'decode_throughput (tokens/s)': 30.05987, 'decode_duration (s)': 4.31193, 'input_token': 512, 'new_token': 128, 'total_token': 640, 'stream': False}