Views
No views yet
pip install transformers==4.57.6 ufal.chu_liu_edmonds==1.0.31curl -sSfL https://hf.co/git-xet/install.sh | sh
2git clone https://huggingface.co/HPLT/hplt_gpt_bert_base_3_0_kor_Hang-UD
3cd hplt_gpt_bert_base_3_0_kor_Hang-UD/1import torch
2
3from lemma_rule import apply_lemma_rule
4from preprocessor import Preprocessor
5from utils import load_model
6
7
8sentences = [
9 "One evening I was walking along a path, the city was on one side and the fjord below.",
10 "I stopped and looked out over the fjord – the sun was setting, and the clouds turning blood red.",
11]
12device = "cuda" if torch.cuda.is_available() else "cpu"
13tokenizer, model, predict_model = load_model(model_path="HPLT/hplt_gpt_bert_base_3_0_kor_Hang", device=device)
14predict_model.eval()
15preprocessor = Preprocessor(tokenizer=tokenizer)
16batch = preprocessor.preprocess(sentences)
17with torch.no_grad():
18 lemma_p, upos_p, xpos_p, feats_p, _, __, dep_p, head_p = predict_model(
19 batch["subwords"].to(device),
20 batch["alignment"].to(device),
21 batch["subword_lengths"].to(device),
22 batch["word_lengths"].to(device),
23 )
24for i in range(len(sentences)):
25 for j, form in enumerate(batch["words"][i]):
26
27 lemma_rule = {
28 rule_type: model["dataset"].lemma_vocab[rule_type].get(p[i, j, :].argmax().item(),
29 model["dataset"].lemma_vocab[rule_type][-1])
30 for rule_type, p in lemma_p.items()
31 }
32 print(form)
33 print(f"lemma {apply_lemma_rule(form, lemma_rule)}")
34 print(f"upos {model['dataset'].upos_vocab[upos_p[i, j, :].argmax().item()]}")
35 print(f"xpos {model['dataset'].xpos_vocab[xpos_p[i, j, :].argmax().item()]}")
36 print(f"feats {model['dataset'].feats_vocab[feats_p[i, j, :].argmax().item()]}")
37 print(f"head {head_p[i, j].item()}")
38 print(f"deprel {model['dataset'].arc_dep_vocab[dep_p[i, j, :].argmax().item()]}")
39 print("_____________________________________________________________________")One
lemma one
upos NUM
xpos CD
feats NumForm=Word|NumType=Card
head 2
deprel nummod
_____________________________________________________________________
evening
lemma evening
upos NOUN
xpos NN
feats Number=Sing
head 5
deprel obl:tmod
_____________________________________________________________________1@inproceedings{charpentier-samuel-2024-bert,
2 title = "{GPT} or {BERT}: why not both?",
3 author = "Charpentier, Lucas Georges Gabriel and
4 Samuel, David",
5 booktitle = "The 2nd BabyLM Challenge at the 28th Conference on Computational Natural Language Learning",
6 month = nov,
7 year = "2024",
8 address = "Miami, FL, USA",
9 publisher = "Association for Computational Linguistics",
10 url = "https://aclanthology.org/2024.conll-babylm.24/",
11 pages = "262--283"
12}1@inproceedings{burchell-etal-2025-expanded,
2 title = "An Expanded Massive Multilingual Dataset for High-Performance Language Technologies ({HPLT})",
3 author = {Burchell, Laurie and
4 de Gibert, Ona and
5 Arefyev, Nikolay and
6 Aulamo, Mikko and
7 Ba{\~n}{\'o}n, Marta and
8 Chen, Pinzhen and
9 Fedorova, Mariia and
10 Guillou, Liane and
11 Haddow, Barry and
12 Haji{\v{c}}, Jan and
13 Helcl, Jind{\v{r}}ich and
14 Henriksson, Erik and
15 Klimaszewski, Mateusz and
16 Komulainen, Ville and
17 Kutuzov, Andrey and
18 Kyt{\"o}niemi, Joona and
19 Laippala, Veronika and
20 M{\ae}hlum, Petter and
21 Malik, Bhavitvya and
22 Mehryary, Farrokh and
23 Mikhailov, Vladislav and
24 Moghe, Nikita and
25 Myntti, Amanda and
26 O{'}Brien, Dayy{\'a}n and
27 Oepen, Stephan and
28 Pal, Proyag and
29 Piha, Jousia and
30 Pyysalo, Sampo and
31 Ram{\'i}rez-S{\'a}nchez, Gema and
32 Samuel, David and
33 Stepachev, Pavel and
34 Tiedemann, J{\"o}rg and
35 Vari{\v{s}}, Du{\v{s}}an and
36 Vojt{\v{e}}chov{\'a}, Tereza and
37 Zaragoza-Bernabeu, Jaume},
38 editor = "Che, Wanxiang and
39 Nabende, Joyce and
40 Shutova, Ekaterina and
41 Pilehvar, Mohammad Taher",
42 booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
43 month = jul,
44 year = "2025",
45 address = "Vienna, Austria",
46 publisher = "Association for Computational Linguistics",
47 url = "https://aclanthology.org/2025.acl-long.854/",
48 doi = "10.18653/v1/2025.acl-long.854",
49 pages = "17452--17485",
50 ISBN = "979-8-89176-251-0",
51 abstract = "Training state-of-the-art large language models requires vast amounts of clean and diverse textual data. However, building suitable multilingual datasets remains a challenge. In this work, we present HPLT v2, a collection of high-quality multilingual monolingual and parallel corpora, extending prior work of the HPLT project. The monolingual portion of the data contains 8T tokens covering 193 languages, while the parallel data contains 380M sentence pairs covering 51 languages. We document the entire data pipeline and release the code to reproduce it. We provide extensive analysis of the quality and characteristics of our data. Finally, we evaluate the performance of language models and machine translation systems trained on HPLT v2, demonstrating its value."
52}1@misc{oepen2025hplt30largescalemultilingual,
2 title={{HPLT 3.0}: {V}ery Large-Scale Multilingual Resources for {LLM} and {MT}. Mono- and Bi-lingual Data, Multilingual Evaluation, and Pre-Trained Models},
3 author={Stephan Oepen and Nikolay Arefev and Mikko Aulamo and Marta Bañón and Maja Buljan and Laurie Burchell and Lucas Charpentier and Pinzhen Chen and Mariia Fedorova and Ona de Gibert and Barry Haddow and Jan Hajič and Jindřich Helcl and Andrey Kutuzov and Veronika Laippala and Zihao Li and Risto Luukkonen and Bhavitvya Malik and Vladislav Mikhailov and Amanda Myntti and Dayyán O'Brien and Lucie Poláková and Sampo Pyysalo and Gema Ramírez Sánchez and Janine Siewert and Pavel Stepachev and Jörg Tiedemann and Teemu Vahtola and Dušan Variš and Fedor Vitiugin and Tea Vojtěchová and Jaume Zaragoza},
4 year={2025},
5 eprint={2511.01066},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL},
8 url={https://arxiv.org/abs/2511.01066},
9}1@article{de-marneffe-etal-2021-universal,
2 title = "{U}niversal {D}ependencies",
3 author = "de Marneffe, Marie-Catherine and
4 Manning, Christopher D. and
5 Nivre, Joakim and
6 Zeman, Daniel",
7 journal = "Computational Linguistics",
8 volume = "47",
9 number = "2",
10 month = jun,
11 year = "2021",
12 address = "Cambridge, MA",
13 publisher = "MIT Press",
14 url = "https://aclanthology.org/2021.cl-2.11/",
15 doi = "10.1162/coli_a_00402",
16 pages = "255--308",
17 abstract = "Universal dependencies (UD) is a framework for morphosyntactic annotation of human language, which to date has been used to create treebanks for more than 100 languages. In this article, we outline the linguistic theory of the UD framework, which draws on a long tradition of typologically oriented grammatical theories. Grammatical relations between words are centrally used to explain how predicate{--}argument structures are encoded morphosyntactically in different languages while morphological features and part-of-speech classes give the properties of words. We argue that this theory is a good basis for crosslinguistically consistent annotation of typologically diverse languages in a way that supports computational natural language understanding as well as broader linguistic studies."
18}1@inproceedings{nivre-etal-2020-universal,
2 title = "{U}niversal {D}ependencies v2: An Evergrowing Multilingual Treebank Collection",
3 author = "Nivre, Joakim and
4 de Marneffe, Marie-Catherine and
5 Ginter, Filip and
6 Haji{\v{c}}, Jan and
7 Manning, Christopher D. and
8 Pyysalo, Sampo and
9 Schuster, Sebastian and
10 Tyers, Francis and
11 Zeman, Daniel",
12 editor = "Calzolari, Nicoletta and
13 B{\'e}chet, Fr{\'e}d{\'e}ric and
14 Blache, Philippe and
15 Choukri, Khalid and
16 Cieri, Christopher and
17 Declerck, Thierry and
18 Goggi, Sara and
19 Isahara, Hitoshi and
20 Maegaard, Bente and
21 Mariani, Joseph and
22 Mazo, H{\'e}l{\`e}ne and
23 Moreno, Asuncion and
24 Odijk, Jan and
25 Piperidis, Stelios",
26 booktitle = "Proceedings of the Twelfth Language Resources and Evaluation Conference",
27 month = may,
28 year = "2020",
29 address = "Marseille, France",
30 publisher = "European Language Resources Association",
31 url = "https://aclanthology.org/2020.lrec-1.497/",
32 pages = "4034--4043",
33 language = "eng",
34 ISBN = "979-10-95546-34-4",
35 abstract = "Universal Dependencies is an open community effort to create cross-linguistically consistent treebank annotation for many languages within a dependency-based lexicalist framework. The annotation consists in a linguistically motivated word segmentation; a morphological layer comprising lemmas, universal part-of-speech tags, and standardized morphological features; and a syntactic layer focusing on syntactic relations between predicates, arguments and modifiers. In this paper, we describe version 2 of the universal guidelines (UD v2), discuss the major changes from UD v1 to UD v2, and give an overview of the currently available treebanks for 90 languages."
36}
