Views
No views yet
pip install jieba1data_dir=/path/to/wenetspeech
2# the data_dir contains:
3# tree -L 2 .
4# .
5# |-- TERMS_OF_ACCESS
6# |-- WenetSpeech.json
7# |-- audio
8# |-- dev
9# |-- test_meeting
10# |-- test_net
11# `-- train
12grep "\"text\":" $data_dir/WenetSpeech.json | sed -e 's/["text: ]*//g' > text.txt
13python -m jieba -d " " text.txt > tokenized.txt
14cat tokenized.txt | awk '{for(i=1;i<=NF;i++)print $i}' | sort | uniq > words.txt