130 lines
3.9 KiB
Python
130 lines
3.9 KiB
Python
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du)
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
import re
|
||
chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]+')
|
||
|
||
|
||
# whether contain chinese character
|
||
def contains_chinese(text):
|
||
return bool(chinese_char_pattern.search(text))
|
||
|
||
|
||
# replace special symbol
|
||
def replace_corner_mark(text):
|
||
text = text.replace('²', '平方')
|
||
text = text.replace('³', '立方')
|
||
return text
|
||
|
||
|
||
# remove meaningless symbol
|
||
def remove_bracket(text):
|
||
text = text.replace('（', '').replace('）', '')
|
||
text = text.replace('【', '').replace('】', '')
|
||
text = text.replace('`', '').replace('`', '')
|
||
text = text.replace("——", " ")
|
||
return text
|
||
|
||
|
||
# spell Arabic numerals
|
||
def spell_out_number(text: str, inflect_parser):
|
||
new_text = []
|
||
st = None
|
||
for i, c in enumerate(text):
|
||
if not c.isdigit():
|
||
if st is not None:
|
||
num_str = inflect_parser.number_to_words(text[st: i])
|
||
new_text.append(num_str)
|
||
st = None
|
||
new_text.append(c)
|
||
else:
|
||
if st is None:
|
||
st = i
|
||
if st is not None and st < len(text):
|
||
num_str = inflect_parser.number_to_words(text[st:])
|
||
new_text.append(num_str)
|
||
return ''.join(new_text)
|
||
|
||
|
||
# split paragrah logic：
|
||
# 1. per sentence max len token_max_n, min len token_min_n, merge if last sentence len less than merge_len
|
||
# 2. cal sentence len according to lang
|
||
# 3. split sentence according to puncatation
|
||
def split_paragraph(text: str, tokenize, lang="zh", token_max_n=80, token_min_n=60, merge_len=20, comma_split=False):
|
||
def calc_utt_length(_text: str):
|
||
if lang == "zh":
|
||
return len(_text)
|
||
else:
|
||
return len(tokenize(_text))
|
||
|
||
def should_merge(_text: str):
|
||
if lang == "zh":
|
||
return len(_text) < merge_len
|
||
else:
|
||
return len(tokenize(_text)) < merge_len
|
||
|
||
if lang == "zh":
|
||
pounc = ['。', '？', '！', '；', '：', '、', '.', '?', '!', ';']
|
||
else:
|
||
pounc = ['.', '?', '!', ';', ':']
|
||
if comma_split:
|
||
pounc.extend(['，', ','])
|
||
|
||
if text[-1] not in pounc:
|
||
if lang == "zh":
|
||
text += "。"
|
||
else:
|
||
text += "."
|
||
|
||
st = 0
|
||
utts = []
|
||
for i, c in enumerate(text):
|
||
if c in pounc:
|
||
if len(text[st: i]) > 0:
|
||
utts.append(text[st: i] + c)
|
||
if i + 1 < len(text) and text[i + 1] in ['"', '”']:
|
||
tmp = utts.pop(-1)
|
||
utts.append(tmp + text[i + 1])
|
||
st = i + 2
|
||
else:
|
||
st = i + 1
|
||
|
||
final_utts = []
|
||
cur_utt = ""
|
||
for utt in utts:
|
||
if calc_utt_length(cur_utt + utt) > token_max_n and calc_utt_length(cur_utt) > token_min_n:
|
||
final_utts.append(cur_utt)
|
||
cur_utt = ""
|
||
cur_utt = cur_utt + utt
|
||
if len(cur_utt) > 0:
|
||
if should_merge(cur_utt) and len(final_utts) != 0:
|
||
final_utts[-1] = final_utts[-1] + cur_utt
|
||
else:
|
||
final_utts.append(cur_utt)
|
||
|
||
return final_utts
|
||
|
||
|
||
# remove blank between chinese character
|
||
def replace_blank(text: str):
|
||
out_str = []
|
||
for i, c in enumerate(text):
|
||
if c == " ":
|
||
if ((text[i + 1].isascii() and text[i + 1] != " ") and
|
||
(text[i - 1].isascii() and text[i - 1] != " ")):
|
||
out_str.append(c)
|
||
else:
|
||
out_str.append(c)
|
||
return "".join(out_str)
|