以下のコードでモデルをロードして使用できます。

!pip install -U bitsandbytes
!pip install -U transformers
!pip install -U accelerate
!pip install -U datasets
# notebookでインタラクティブな表示を可能とする（ただし、うまく動かない場合あり）
!pip install ipywidgets --upgrade

# Hugging Faceで取得したTokenをこちらに貼る。
HF_TOKEN = <your key>

# モデルのID
model_name = "Rakushaking/llm-jp-3-13b-finetune-it"

# QLoRA config
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=False,
)

# Load model
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    token = HF_TOKEN
)

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, token = HF_TOKEN)

prompt = f"""### 指示：<your question>
### 回答：

tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
with torch.no_grad():
  outputs = model.generate(
  tokenized_input,
  max_new_tokens=100,
  do_sample=False,
  repetition_penalty=1.2
      )[0]
output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)

print(output)

推論手法②（RAG拡張生成）

アーキテクチャ

python

#自分の作成したモデルのIDをこちらに貼る。
model_name = "Rakushaking/llm-jp-3-13b-it"

from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
)
import torch
from tqdm import tqdm
import json

#QLoRA config
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=False,
)

#Load model
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    token = HF_TOKEN
)

#Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, token = HF_TOKEN)

import langchain
from langchain.embeddings import HuggingFaceEmbeddings
from llama_index.core import ServiceContext, SQLDatabase, VectorStoreIndex
from typing import Any, List

#埋め込みクラスにqueryを付加
class HuggingFaceQueryEmbeddings(HuggingFaceEmbeddings):
    def __init__(self, **kwargs: Any):
        super().__init__(**kwargs)

    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        return super().embed_documents(["query: " + text for text in texts])

    def embed_query(self, text: str) -> List[float]:
        return super().embed_query("query: " + text)

#ベクトル化する準備
embedding = langchain.embeddings.HuggingFaceEmbeddings(
    model_name="intfloat/multilingual-e5-base",
    #model_kwargs=model_kwargs,
    #encode_kwargs=encode_kwargs
)

from transformers import pipeline
from langchain.llms import HuggingFacePipeline

#パイプラインの準備
pipe = pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
    max_new_tokens=512
)

from langchain_community.document_loaders import JSONLoader

loader = JSONLoader(
    file_path="<YOUR DATABASE>",
    jq_schema=".summary",
    text_content=False,
    json_lines=True,  # JSONL形式でファイルを読み込む
)

docs = loader.load()
print(docs[0])

import langchain.text_splitter

#読込した内容を分割する
text_splitter = langchain.text_splitter.RecursiveCharacterTextSplitter(
    chunk_size=1024,
    chunk_overlap=0,
)
docs = text_splitter.split_documents(docs)

#FAISS indexの作成
from langchain.vectorstores import FAISS
vectorstore = FAISS.from_documents(docs, embedding)


retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 2})

from langchain.prompts import ChatPromptTemplate
from langchain_community.vectorstores import Chroma
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnableLambda, RunnablePassthrough

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

#Modified chain definition:
chain = (
    RunnableLambda(lambda x: {"context": format_docs(retriever.get_relevant_documents(x)), "query": x})
    | prompt
    | RunnableLambda(lambda x: x.to_string())  # Convert StringPromptValue to string
    | pipe
    | RunnableLambda(lambda x: x[0]["generated_text"] if isinstance(x, list) and x else x["generated_text"])  # Extract generated text from the output of the pipe
    | StrOutputParser()
)

#Invoke the chain with the question string directly, not a dictionary
res = chain.invoke(question)

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

def RAG(user_prompt):
  #プロンプトを準備
  template =  """
  <bos><start_of_turn>###指示
  次の文脈を参考にして回答してください。
  ただし、参考情報が質問に関係ない場合は、参考情報を無視して回答を生成してください。
  また、回答の際は、同じ単語や話題を繰り返さないでください。
  {context}
  <end_of_turn><start_of_turn>###質問
  {query}
   <end_of_turn><start_of_turn>###回答
   """
  prompt = langchain.prompts.PromptTemplate.from_template(template) # Corrected indentation

  #Modified chain definition:
  chain = (
      RunnableLambda(lambda x: {"context": format_docs(retriever.get_relevant_documents(x)), "query": x})
      | prompt
      | RunnableLambda(lambda x: x.to_string())  # Convert StringPromptValue to string
      | pipe
      | RunnableLambda(lambda x: x[0]["generated_text"] if isinstance(x, list) and x else x["generated_text"])  # Extract generated text from the output of the pipe
      | StrOutputParser()
  )

  res = chain.invoke(user_prompt)
  result = res.split("###回答")[-1].replace("\n", "")
  return result


  def llm(user_prompt):

  prompt = f"""### 指示
  {user_prompt}
  #回答：
  """

  tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
  with torch.no_grad():
      outputs = model.generate(
          tokenized_input,
          max_new_tokens=512,
          do_sample=False,
          use_cache = True,
          repetition_penalty=1.2,
          temperature=0.1,
          pad_token_id=tokenizer.eos_token_id
      )[0]
  output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
  return output

  #results.append({"task_id": data["task_id"], "input": input, "output": output})

def judge_score_llm(user_prompt):
  scores = [item[1] for item in vectorstore.similarity_search_with_score(user_prompt)]
  max_score = max(scores)

  if max_score < 0.20:
    print("###RAG####")
    return RAG(user_prompt),

  else:
    print("####LLMより回答生成")
    return llm(user_prompt),
    #return llm(user_prompt),print("####LLMより回答生成")

def judge_score_llm(user_prompt):
    scores = [item[1] for item in vectorstore.similarity_search_with_score(user_prompt)]
    avg_score = sum(scores) / len(scores) if scores else 0  # スコアの平均を計算

    if avg_score < 0.25:  # 平均スコアが 0.20 以下の場合
        print("###RAG####")
        return RAG(user_prompt)
    else:
        print("####LLMより回答生成")
        return llm(user_prompt)
        
#データセットの読み込み。

datasets = []
with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
    item = ""
    for line in f:
      line = line.strip()
      item += line
      if item.endswith("}"):
        datasets.append(json.loads(item))
        item = ""

”””python