Views
No views yet
1models:
2 - model: liminerity/M7-7b
3 # no parameters necessary for base model
4 - model: automerger/YamShadow-7B
5 parameters:
6 weight: 0.3
7 density: 0.5
8 - model: mlabonne/AlphaMonarch-7B
9 parameters:
10 weight: 0.2
11 density: 0.5
12 - model: automerger/OgnoExperiment27-7B
13 parameters:
14 weight: 0.2
15 density: 0.5
16 - model: Kukedlc/Jupiter-k-7B-slerp
17 parameters:
18 weight: 0.3
19 density: 0.5
20merge_method: dare_ties
21base_model: liminerity/M7-7b
22
23parameters:
24 int8_mask: true
25 normalize: true
26dtype: bfloat161# Requirements
2!pip install -qU transformers accelerate bitsandbytes
3
4# Imports & settings
5from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
6import warnings
7import os
8os.environ["TOKENIZERS_PARALLELISM"] = "false"
9warnings.filterwarnings('ignore')
10
11# Model & Tokenizer
12MODEL_NAME = "Kukedlc/NeuralShiva-7B-DT"
13model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map='cuda:1', load_in_4bit=True)
14tok = AutoTokenizer.from_pretrained(MODEL_NAME)
15
16# Inference
17prompt = "I want you to generate a theory that unites quantum mechanics with the theory of relativity and cosmic consciousness"
18inputs = tok([prompt], return_tensors="pt").to('cuda')
19streamer = TextStreamer(tok)
20
21# Despite returning the usual output, the streamer will also print the generated text to stdout.
22_ = model.generate(**inputs, streamer=streamer, max_new_tokens=512, do_sample=True, num_beams=1, top_p=0.9, temperature=0.7)
231!pip install -qU transformers bitsandbytes accelerate
2
3from transformers import AutoTokenizer
4import transformers
5import torch
6
7model = "Kukedlc/NeuralShiva-7B-DT"
8
9tokenizer = AutoTokenizer.from_pretrained(model)
10pipeline = transformers.pipeline(
11 "text-generation",
12 model=model,
13 model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
14)
15
16messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
17prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
18outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
19print(outputs[0]["generated_text"])