import json

with open("verbalist/datasets/RyokoAI_ShareGPT52K/sg_90k_part1.json") as f: dataset1 = json.load(f) with open("verbalist/datasets/RyokoAI_ShareGPT52K/sg_90k_part2.json") as f: dataset2 = json.load(f) dataset = dataset1 + dataset2

conversation_field = "conversations"

import re import regex import hashlib

def filter_string(string): has = True… See the full description on the dataset page: https://huggingface.co/datasets/dim/sharegpt_short_en_3k.