Fix py3 compatibility in preprocessing.text
This commit is contained in:
parent
d133080942
commit
43206aeb14
@ -5,11 +5,16 @@
|
|||||||
'''
|
'''
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
|
|
||||||
import string
|
import string, sys
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from six.moves import range
|
from six.moves import range
|
||||||
from six.moves import zip
|
from six.moves import zip
|
||||||
|
|
||||||
|
if sys.version_info < (3,):
|
||||||
|
maketrans = string.maketrans
|
||||||
|
else:
|
||||||
|
maketrans = str.maketrans
|
||||||
|
|
||||||
def base_filter():
|
def base_filter():
|
||||||
f = string.punctuation
|
f = string.punctuation
|
||||||
f = f.replace("'", '')
|
f = f.replace("'", '')
|
||||||
@ -21,7 +26,7 @@ def text_to_word_sequence(text, filters=base_filter(), lower=True, split=" "):
|
|||||||
'''
|
'''
|
||||||
if lower:
|
if lower:
|
||||||
text = text.lower()
|
text = text.lower()
|
||||||
text = text.translate(string.maketrans(filters, split*len(filters)))
|
text = text.translate(maketrans(filters, split*len(filters)))
|
||||||
seq = text.split(split)
|
seq = text.split(split)
|
||||||
return [_f for _f in seq if _f]
|
return [_f for _f in seq if _f]
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user