Views
No views yet
pip install hf_hub_ctranslate2>=1.0.0 ctranslate2>=3.13.0compute_type=int8_float16 for device="cuda"compute_type=int8 for device="cpu"1from hf_hub_ctranslate2 import TranslatorCT2fromHfHub, GeneratorCT2fromHfHub
2
3model_name = "michaelfeil/ct2fast-dolly-v2-12b"
4model = GeneratorCT2fromHfHub(
5 # load in int8 on CUDA
6 model_name_or_path=model_name,
7 device="cuda",
8 compute_type="int8_float16"
9)
10outputs = model.generate(
11 text=["How do you call a fast Flan-ingo?", "User: How are you doing?"],
12)
13print(outputs)1# from https://huggingface.co/databricks/dolly-v2-12b
2def encode_prompt(instruction):
3 INSTRUCTION_KEY = "### Instruction:"
4 RESPONSE_KEY = "### Response:"
5 END_KEY = "### End"
6 INTRO_BLURB = (
7 "Below is an instruction that describes a task. Write a response that appropriately completes the request."
8 )
9
10 # This is the prompt that is used for generating responses using an already trained model. It ends with the response
11 # key, where the job of the model is to provide the completion that follows it (i.e. the response itself).
12 PROMPT_FOR_GENERATION_FORMAT = """{intro}
13 {instruction_key}
14 {instruction}
15 {response_key}
16 """.format(
17 intro=INTRO_BLURB,
18 instruction_key=INSTRUCTION_KEY,
19 instruction="{instruction}",
20 response_key=RESPONSE_KEY,
21 )
22 return PROMPT_FOR_GENERATION_FORMAT.format(instruction=instruction)