44 lines
1.1 KiB
Python
44 lines
1.1 KiB
Python
|
||
import re
|
||
|
||
# 非打印字符
|
||
NON_PRINTING_CHARS_RE = re.compile(
|
||
f"[{''.join(map(chr, list(range(0, 32)) + list(range(127, 160))))}]"
|
||
)
|
||
|
||
class DocTokenizer():
|
||
'''
|
||
文档text处理器。
|
||
'''
|
||
|
||
def __init__(self):
|
||
pass
|
||
|
||
def doc_process(self, text):
|
||
'''
|
||
去除多余换行、去掉每行非打印字符和开头结尾空格
|
||
'''
|
||
# 去除多余换行
|
||
text = self.remove_excess_lines(text)
|
||
# 将文本拆分成行
|
||
lines = text.split("\n")
|
||
# 去掉每一行的开头和结尾的空格
|
||
lines = [self.remove_non_printing_char_line(
|
||
line.strip()) for line in lines]
|
||
# 将行重新组合成文本
|
||
text_new = "\n".join(lines)
|
||
return text_new
|
||
|
||
def remove_excess_lines(self, text):
|
||
'''
|
||
将2个以上的换行符替换为2个，html解析text时会产生大量换行\n
|
||
'''
|
||
pattern = r'\n\n+'
|
||
return re.sub(pattern, '\n\n', text)
|
||
|
||
def remove_non_printing_char_line(self, text):
|
||
'''
|
||
去除每一行的非打印字符
|
||
'''
|
||
return NON_PRINTING_CHARS_RE.sub("", text)
|