Views
No views yet
| Bleu-1 | Bleu-2 | Bleu-3 | Bleu-4 | METEOR | ROUGE-L | CIDEr | SPICE |
|---|---|---|---|---|---|---|---|
| 0.74 | 0.62 | 0.50 | 0.40 | 0.31 | 0.65 | 1.73 | 0.21 |
1from transformers.pytorch_transformers import BertConfig, BertTokenizer
2from oscar.modeling.modeling_bert import BertForImageCaptioning
3from oscar.wrappers import OscarTensorizer
4
5ckpt = "path/to/the/checkpoint"
6device = "cuda" if torch.cuda.is_available() else "cpu"
7
8# original code
9config = BertConfig.from_pretrained(ckpt)
10tokenizer = BertTokenizer.from_pretrained(ckpt)
11model = BertForImageCaptioning.from_pretrained(ckpt, config=config).to(device)
12
13# This takes care of the preprocessing
14tensorizer = OscarTensorizer(tokenizer=tokenizer, device=device)
15
16# numpy-arrays with shape (1, num_boxes, feat_size)
17# feat_size is 2054 by default in VinVL
18visual_features = torch.from_numpy(feat_obj).to(device).unsqueeze(0)
19
20# labels are usually extracted by the features extractor
21labels = [['boat', 'boat', 'boat', 'bottom', 'bush', 'coat', 'deck', 'deck', 'deck', 'dock', 'hair', 'jacket']]
22
23inputs = tensorizer.encode(visual_features, labels=labels)
24outputs = model(**inputs)
25
26pred = tensorizer.decode(outputs)
27
28# the output looks like this:
29# pred = {0: [{'caption': 'He is sailing', 'conf': 0.7070220112800598]}1@inproceedings{cafagna2023hl,
2 title={{HL} {D}ataset: {V}isually-grounded {D}escription of {S}cenes, {A}ctions and
3{R}ationales},
4 author={Cafagna, Michele and van Deemter, Kees and Gatt, Albert},
5 booktitle={Proceedings of the 16th International Natural Language Generation Conference (INLG'23)},
6address = {Prague, Czech Republic},
7 year={2023}
8}1
2@misc{han2021image,
3 title={Image Scene Graph Generation (SGG) Benchmark},
4 author={Xiaotian Han and Jianwei Yang and Houdong Hu and Lei Zhang and Jianfeng Gao and Pengchuan Zhang},
5 year={2021},
6 eprint={2107.12604},
7 archivePrefix={arXiv},
8 primaryClass={cs.CV}
9}
10
11@inproceedings{zhang2021vinvl,
12 title={Vinvl: Revisiting visual representations in vision-language models},
13 author={Zhang, Pengchuan and Li, Xiujun and Hu, Xiaowei and Yang, Jianwei and Zhang, Lei and Wang, Lijuan and Choi, Yejin and Gao, Jianfeng},
14 booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
15 pages={5579--5588},
16 year={2021}
17}