Views
No views yet
pip install -U huggingface_hub["cli"]hf download savvadesogle/T-pro-it-2.1-int8-ov --local-dir ./T-pro-it-2.1-int8-ovoptimum-cli export openvino with the following parameters:1optimum-cli export openvino ^
2 --model ./T-pro-it-2.1 ^
3 --task text-generation-with-past ^
4 --weight-format int8 ^
5 ./T-pro-it-2.1-int8-ovset OPENARC_API_KEY=BIG_KEY
openarc serve start --host 127.0.0.1set OPENARC_API_KEY=BIG_KEY
openarc add --model-name T-pro-it-2.1 --model-path ./T-pro-it-2.1-int8-ov --engine ovgenai --model-type llm --device GPU
openarc load T-pro-it-2.1
openarc bench T-pro-it-2.1http://127.0.0.1:8000/v1/chat/completionscurl http://127.0.0.1:8000/v1/chat/completions ^
-H "Content-Type: application/json" ^
-H "Authorization: Bearer BIG_KEY" ^
-d "{\"model\":\"T-pro-it-2.1-int8-ov\",\"messages\":[{\"role\":\"user\",\"content\":\"Расскажи анекдот\"}],\"temperature\":0.7,\"max_tokens\":128}"openarc bench T-pro-it-2.1-int8-ov