Fixing full width spaces

This commit is contained in:
The Gears of Progress 2026-01-29 23:57:21 -05:00
parent 5447b39569
commit 5536c5690b

View File

@ -225,9 +225,11 @@ def convert_item(ogDict, lang):
# Change all the punctuation marks followed by spaces into being followed by _ .
# These will end up being replaced by spaces anyway in the end (but ignore the error)
spaces = [' ', ' ']
puncts = ['.', '?', '!', '', '', '']
for punct in puncts:
line = line.replace(punct + " ", punct + "_")
for space in spaces:
for punct in puncts:
line = line.replace(punct + space, punct + "_")
split_sents = split_into_sentences(line)
index = 0