Views
No views yet
1import torch
2import transformers
3
4# prepare the LLaMA 2 model
5model_name = "tifa-benchmark/llama2_tifa_question_generation"
6pipeline = transformers.pipeline(
7 "text-generation",
8 model=model_name,
9 torch_dtype=torch.float16,
10 device_map="auto",
11)
12
13
14# formating prompt following LLaMA 2 style
15def create_qg_prompt(caption):
16 INTRO_BLURB = "Given an image description, generate one or two multiple-choice questions that verifies if the image description is correct.\nClassify each concept into a type (object, human, animal, food, activity, attribute, counting, color, material, spatial, location, shape, other), and then generate a question for each type.\n"
17 formated_prompt = f"<s>[INST] <<SYS>>\n{INTRO_BLURB}\n<</SYS>>\n\n"
18 formated_prompt += f"Description: {caption} [/INST] Entities:"
19 return formated_prompt
20
21
22test_caption = "a blue rabbit and a red plane"
23
24# create prompt
25prompt = create_qg_prompt(text_caption)
26
27# text completion
28sequences = pipeline(
29 prompt, do_sample=False, num_beams=5, num_return_sequences=1, max_length=512)
30output = sequences[0]['generated_text'][len(prompt):]
31output = output.split('\n\n')[0]
32
33# output
34print(output)
35
36#### Expected output ###
37# rabbit, plane
38# Activites:
39# Colors: blue, red
40# Counting:
41# Other attributes:
42# About rabbit (animal):
43# Q: is this a rabbit?
44# Choices: yes, no
45# A: yes
46# About rabbit (animal):
47# Q: what animal is in the picture?
48# Choices: rabbit, dog, cat, fish
49# A: rabbit
50# About plane (object):
51# Q: is this a plane?
52# Choices: yes, no
53# A: yes
54# About plane (object):
55# Q: what type of vehicle is this?
56# Choices: plane, car, motorcycle, bus
57# A: plane
58# About blue (color):
59# Q: is the rabbit blue?
60# Choices: yes, no
61# A: yes
62# About blue (color):
63# Q: what color is the rabbit?
64# Choices: blue, red, yellow, green
65# A: blue
66# About red (color):
67# Q: is the plane red?
68# Choices: yes, no
69# A: yes
70# About red (color):
71# Q: what color is the plane?
72# Choices: red, blue, yellow, green
73# A: red1from tifascore import get_llama2_pipeline, get_llama2_question_and_answers
2
3pipeline = get_llama2_pipeline("tifa-benchmark/llama2_tifa_question_generation")
4
5print(get_llama2_question_and_answers(pipeline, "a blue rabbit and a red plane"))
6
7#### Expected output ###
8# [{'caption': 'a blue rabbit and a red plane', 'element': 'rabbit', 'question': 'what animal is in the picture?', 'choices': ['rabbit', 'dog', 'cat', 'fish'], 'answer': 'rabbit', 'element_type': 'animal/human'}, {'caption': 'a blue rabbit and a red plane', 'element': 'plane', 'question': 'is this a plane?', 'choices': ['yes', 'no'], 'answer': 'yes', 'element_type': 'object'}, {'caption': 'a blue rabbit and a red plane', 'element': 'plane', 'question': 'what type of vehicle is this?', 'choices': ['plane', 'car', 'motorcycle', 'bus'], 'answer': 'plane', 'element_type': 'object'}, {'caption': 'a blue rabbit and a red plane', 'element': 'blue', 'question': 'is the rabbit blue?', 'choices': ['yes', 'no'], 'answer': 'yes', 'element_type': 'color'}, {'caption': 'a blue rabbit and a red plane', 'element': 'blue', 'question': 'what color is the rabbit?', 'choices': ['blue', 'red', 'yellow', 'green'], 'answer': 'blue', 'element_type': 'color'}, {'caption': 'a blue rabbit and a red plane', 'element': 'red', 'question': 'is the plane red?', 'choices': ['yes', 'no'], 'answer': 'yes', 'element_type': 'color'}, {'caption': 'a blue rabbit and a red plane', 'element': 'red', 'question': 'what color is the plane?', 'choices': ['red', 'blue', 'yellow', 'green'], 'answer': 'red', 'element_type': 'color'}]@article{hu2023tifa,
title={Tifa: Accurate and interpretable text-to-image faithfulness evaluation with question answering},
author={Hu, Yushi and Liu, Benlin and Kasai, Jungo and Wang, Yizhong and Ostendorf, Mari and Krishna, Ranjay and Smith, Noah A},
journal={arXiv preprint arXiv:2303.11897},
year={2023}
}