import os
import json
import pandas as pd
folder_path = "双语数据\史记\七十列传"
def get_files(folder_path):
subfolders = [f.path for f in os.scandir(folder_path) if f.is_dir()]
print(subfolders)
dataset = []
source_file = "source.txt"
target_file = "target.txt"
for x in subfolders:
with open(os.path.join(x,source_file) , "r", encoding="utf-8") as f:
source_content = f.read()
with open(os.path.join(x,target_file), "r", encoding="utf-8") as f:
target_content = f.read()… See the full description on the dataset page:
https://huggingface.co/datasets/yuwei1289/LLMclass.
