random.seed(seedn)
sample_ratio = 0.5
data_path = '/home/rokabe/data2/cava/data/solid-state_dataset_2019-06-27_upd.json' # path to the inorganic crystal synthesis data (json)
data = json.load(open(data_path, 'r'))
num_sample = int(len(data)*sample_ratio)
separator=' || '
cut = ';'
rand_indices = random.sample(range(len(data)), num_sample)
data1 = [data[i] for i in rand_indices]
dataset = Dataset_Ceq2Ope_3(data1, index=None, te_ratio=0.1, separator=separator, cut=cut).dataset
hf_model = "distilbert-base-uncased"
model_name = hf_usn + '/ope_bert_v1.1'# '/syn_distilgpt2_v2'
tk_model = hf_model #"Dagobert42/gpt2-finetuned-material-synthesis"#'m3rg-iitd/matscibert'##hf_model # set tokenizer model loaded from HF (usually same as hf_model)
load_pretrained=False # If True, load the model from 'model_name'. Else, load the pre-trained model from hf_model.
pad_tokenizer=False
save_indices = True
rm_ckpts = True
