# pip install git+https://github.com/huggingface/transformers.git
# pip install git+https://github.com/huggingface/peft.git

import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig
)
from peft import PeftModel

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    quantization_config=BitsAndBytesConfig(load_in_8bit=True),
    device_map="auto",
)

tokenizer = AutoTokenizer.from_pretrained("Equall/Saul-7B-Base")
tokenizer.pad_token = tokenizer.eos_token

model = PeftModel.from_pretrained(
            model,
            "auslawbench/Cite-Llama-3.1-8B",
            device_map="auto",
            torch_dtype=torch.bfloat16,
        )
model.eval()

fine_tuned_prompt = """
### Instruction:
{}

### Input:
{}

### Response:
{}"""

example_input="Many of ZAR’s grounds of appeal related to fact finding. Drawing on principles set down in several other courts and tribunals, the Appeal Panel summarised the circumstances in which leave may be granted for a person to appeal from findings of fact: <CASENAME> at [84]."
model_input = fine_tuned_prompt.format("Predict the name of the case that needs to be cited in the text and explain why it should be cited.", example_input, '')
inputs = tokenizer(model_input, return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, max_new_tokens=256, temperature=1.0)
output = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(output.split("### Response:")[1].strip().split('>')[0] + '>')

Citation

BibTeX:

@misc{shareghi2024auslawcite,
      title={Methods for Legal Citation Prediction in the Age of LLMs: An Australian Law Case Study}, 
      author={Ehsan Shareghi, Jiuzhou Han, Paul Burgess},
      year={2024},
      eprint={arXiv:2412.06272},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}