Views
No views yet
pipeline task for text-generation for 🤗 Inference Endpoints for LLM inference using bitsandbytes quantization. The code for the customized pipeline is in the pipeline.py.1{
2 "inputs": "# load distilbert model and initialize text-classification pipeline\nmodel_id = 'distil",
3 "parameters": {
4 "top_k": 100,
5 "max_length": 64,
6 "early_stopping": true,
7 "do_sample": true,
8 "eos_token_id": 50256,
9 }
10}requests.1import json
2from typing import List
3import requests as r
4import base64
5ENDPOINT_URL = ""
6HF_TOKEN = ""
7
8parameters={
9 "top_k": 100,
10 "max_length": 64,
11 "early_stopping": True,
12 "do_sample": True,
13 "eos_token_id": 50256,
14 }
15
16def predict(code_snippet:str=None):
17 payload = {"inputs": code_snippet,"parameters": parameters}
18 response = r.post(
19 ENDPOINT_URL, headers={"Authorization": f"Bearer {HF_TOKEN}"}, json=payload
20 )
21 return response.json()
22prediction = predict(
23 code_snippet="# load distilbert model and initialize text-classification pipeline\nmodel_id = 'distil"
24){'generated_text': "# load distilbert model and initialize text-classification pipeline\nmodel_id = 'distilbert-base-uncased'\nmodel_url = 'https://tfhub.dev/tensorflow/small_bert/1'\n\nmodel_dir = './distilBERT'"}